From 3ba759ea6f9172156ab71c826c19f071c93f9af3 Mon Sep 17 00:00:00 2001 From: lsabor Date: Thu, 7 May 2026 14:09:01 -0700 Subject: [PATCH 01/11] use metac_name for retrieving metaculus api tokens --- .github/workflows/run-bot-aib-tournament.yaml | 1 + .github/workflows/run-bot-launcher.yaml | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 9b16dc76..fb849ce7 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -1545,6 +1545,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_UNIFORM_PROBABILITY_BOT_TOKEN" + metac_name: "metac-uniform-probability-bot" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} diff --git a/.github/workflows/run-bot-launcher.yaml b/.github/workflows/run-bot-launcher.yaml index 8cb249fa..c11bcba9 100644 --- a/.github/workflows/run-bot-launcher.yaml +++ b/.github/workflows/run-bot-launcher.yaml @@ -4,6 +4,10 @@ on: bot_name: required: true type: string + metac_name: + required: false + type: string + default: "" cache_key: required: false type: string @@ -119,9 +123,9 @@ jobs: if [ -n "$RAW_TOKEN" ]; then echo "METACULUS_TOKEN=$RAW_TOKEN" >> $GITHUB_ENV else - TOKEN=$(echo "$METACULUS_TOKENS" | jq -r --arg key "$BOT_NAME" '.[$key] // empty') + TOKEN=$(echo "$METACULUS_TOKENS" | jq -r --arg key "$METAC_NAME" '.[$key] // empty') if [ -z "$TOKEN" ]; then - echo "ERROR: No token found for $BOT_NAME in METACULUS_TOKENS" >&2 + echo "ERROR: No token found for $METAC_NAME in METACULUS_TOKENS" >&2 exit 1 fi echo "METACULUS_TOKEN=$TOKEN" >> $GITHUB_ENV @@ -129,7 +133,7 @@ jobs: env: RAW_TOKEN: ${{ secrets.INPUT_METACULUS_TOKEN }} METACULUS_TOKENS: ${{ secrets.INPUT_METACULUS_TOKENS }} - BOT_NAME: ${{ inputs.bot_name }} + METAC_NAME: ${{ inputs.metac_name }} - name: Run bot run: | From 226973897b3ed59f89d2e23cd85e27a0d3f09387 Mon Sep 17 00:00:00 2001 From: lsabor Date: Sat, 25 Apr 2026 16:52:26 -0700 Subject: [PATCH 02/11] add new bots --- .github/workflows/run-bot-aib-tournament.yaml | 206 ++++++++++++++++++ run_bots.py | 95 ++++++++ 2 files changed, 301 insertions(+) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index fb849ce7..845b827e 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -91,6 +91,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4_1_HIGH" + metac_name: "metac-grok-4-1-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_HIGH }} @@ -104,6 +105,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4_1" + metac_name: "metac-grok-4-1+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1 }} @@ -112,6 +114,149 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + #################################### April 2026 new bots #################################### + + bot_claude_sonnet_4_6: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_CLAUDE_SONNET_4_6" + metac_name: "metac-claude-sonnet-4-6" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + + bot_qwen_3_5: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_QWEN_3_5" + metac_name: "metac-qwen-3-5" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gemini_3_1_pro: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GEMINI_3_1_PRO" + metac_name: "metac-gemini-3-1-pro" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_5: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_5" + metac_name: "metac-gpt-5-5" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_4: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_4" + metac_name: "metac-gpt-5-4" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_4_mini: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_4_MINI" + metac_name: "metac-gpt-5-4-mini" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_4_nano: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_4_NANO" + metac_name: "metac-gpt-5-4-nano" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_3: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_3" + metac_name: "metac-gpt-5-3" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_3_instant: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_3_INSTANT" + metac_name: "metac-gpt-5-3-instant" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_minimax_m2_7: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_MINIMAX_M2_7" + metac_name: "metac-minimax-m2-7" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + #################################### February 2026 new bots #################################### bot_claude_opus_4_6_high_32k: @@ -119,6 +264,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_OPUS_4_6_HIGH_32K" + metac_name: "metac-claude-opus-4-6-high-32k+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_6_HIGH_32K }} @@ -147,6 +293,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_HAIKU_4_5" + metac_name: "metac-claude-haiku-4-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_HAIKU_4_5 }} @@ -161,6 +308,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_QWEN3_MAX_THINKING" + metac_name: "metac-qwen3-max-thinking+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN3_MAX_THINKING }} @@ -187,6 +335,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_KIMI_K2_5_HIGH" + metac_name: "metac-kimi-k2-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_5_HIGH }} @@ -200,6 +349,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GLM_5" + metac_name: "metac-glm-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_5 }} @@ -215,6 +365,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_OPUS_4_5_HIGH_32K" + metac_name: "metac-claude-opus-4-5-high-32k+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5_HIGH_32K }} @@ -229,6 +380,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_OPUS_4_5" + metac_name: "metac-claude-opus-4-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5 }} @@ -243,6 +395,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_2_HIGH" + metac_name: "metac-gpt-5-2-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2_HIGH }} @@ -257,6 +410,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_2" + metac_name: "metac-gpt-5-2+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2 }} @@ -284,6 +438,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GEMINI_3_FLASH" + metac_name: "metac-gemini-3-flash+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_3_FLASH }} @@ -298,6 +453,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GLM_4_6" + metac_name: "metac-glm-4-6+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_4_6 }} @@ -325,6 +481,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_KIMI_K2_HIGH" + metac_name: "metac-kimi-k2-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_HIGH }} @@ -338,6 +495,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_1_HIGH" + metac_name: "metac-gpt-5-1-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1_HIGH }} @@ -352,6 +510,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_1" + metac_name: "metac-gpt-5-1+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1 }} @@ -393,6 +552,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4_1_FAST_HIGH" + metac_name: "metac-grok-4-1-fast-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST_HIGH }} @@ -407,6 +567,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4_1_FAST" + metac_name: "metac-grok-4-1-fast+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST }} @@ -447,6 +608,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_4_5_SONNET_HIGH" + metac_name: "metac-claude-4-5-sonnet-high-32k+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET_HIGH }} @@ -461,6 +623,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_4_5_SONNET" + metac_name: "metac-claude-4-5-sonnet+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET }} @@ -475,6 +638,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_QWEN_3_MAX" + metac_name: "metac-qwen-3-max+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN_3_MAX }} @@ -488,6 +652,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_3_2_REASONING" + metac_name: "metac-deepseek-3-2-reasoning+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_3_2_REASONING }} @@ -513,6 +678,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4_FAST_HIGH" + metac_name: "metac-grok-4-fast-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_FAST_HIGH }} @@ -569,6 +735,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_MINI" + metac_name: "metac-gpt-5-mini+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_MINI }} @@ -583,6 +750,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_NANO" + metac_name: "metac-gpt-5-nano+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_NANO }} @@ -636,6 +804,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_4" + metac_name: "metac-grok-4+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4 }} @@ -650,6 +819,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_KIMI_K2" + metac_name: "metac-kimi-k2+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2 }} @@ -676,6 +846,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_VARIANCE_TEST" + metac_name: "metac-deepseek-r1+asknews(variance-test)" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_VARIANCE_TEST }} @@ -689,6 +860,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_OSS_120B" + metac_name: "metac-gpt-oss-120b+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_OSS_120B }} @@ -702,6 +874,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_ZAI_GLM_4_5" + metac_name: "metac-zai-glm-4-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_ZAI_GLM_4_5 }} @@ -715,6 +888,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_V3_1_REASONING" + metac_name: "metac-deepseek-v3-1-reasoning+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_V3_1_REASONING }} @@ -728,6 +902,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_V3_1" + metac_name: "metac-deepseek-v3-1+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_V3_1 }} @@ -741,6 +916,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_V3_1_VARIANCE_TEST_1" + metac_name: "metac-deepseek-v3-1+asknews(variance-test-1)" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_V3_1_VARIANCE_TEST_1 }} @@ -754,6 +930,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_V3_1_VARIANCE_TEST_2" + metac_name: "metac-deepseek-v3-1+asknews(variance-test-2)" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_V3_1_VARIANCE_TEST_2 }} @@ -812,6 +989,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GEMINI_2_5_PRO_GROUNDING" + metac_name: "metac-gemini-2-5-pro-grounding[research-only]" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_PRO_GROUNDING }} @@ -824,6 +1002,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_ASKNEWS_DEEPNEWS" + metac_name: "metac-asknews-deepnews[research-only]" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_ASKNEWS_DEEPNEWS }} @@ -870,6 +1049,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_EXA_ONLINE_RESEARCH_ONLY" + metac_name: "metac-deepseek-r1-exa-online[research-only-bot]" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_EXA_ONLINE_RESEARCH_ONLY }} @@ -882,6 +1062,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_PLUS_EXA_ONLINE" + metac_name: "metac-deepseek-r1+deepseek-r1-exa-online" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_PLUS_EXA_ONLINE }} @@ -927,6 +1108,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_NO_RESEARCH" + metac_name: "metac-deepseek-r1+no-research" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_NO_RESEARCH }} @@ -939,6 +1121,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_4_1_OPTIMIZED_PROMPT" + metac_name: "metac-gpt-4-1+asknews[optimized-prompt]" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_4_1_OPTIMIZED_PROMPT }} @@ -953,6 +1136,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_4_1_NANO_OPTIMIZED_PROMPT" + metac_name: "metac-gpt-4-1-nano+asknews[optimized-prompt]" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_4_1_NANO_OPTIMIZED_PROMPT }} @@ -1038,6 +1222,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_SONAR_PRO" + metac_name: "metac-deepseek-r1+sonar-pro" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_SONAR_PRO }} @@ -1050,6 +1235,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_SONAR" + metac_name: "metac-deepseek-r1+sonar" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_SONAR }} @@ -1073,6 +1259,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_SONAR_REASONING_PRO" + metac_name: "metac-deepseek-r1+sonar-reasoning-pro" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_SONAR_REASONING_PRO }} @@ -1108,6 +1295,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW" + metac_name: "metac-deepseek-r1+gpt-4o-search-preview" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW }} @@ -1120,6 +1308,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING" + metac_name: "metac-deepseek-r1+gemini-2-5-pro-grounding" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING }} @@ -1143,6 +1332,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_ASK_EXA_PRO" + metac_name: "metac-deepseek-r1+exa-answer" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_ASK_EXA_PRO }} @@ -1181,6 +1371,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_O3_TOKEN" + metac_name: "metac-o3+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O3_TOKEN }} @@ -1195,6 +1386,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_O4_MINI_HIGH_TOKEN" + metac_name: "metac-o4-mini-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_HIGH_TOKEN }} @@ -1209,6 +1401,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_O4_MINI_TOKEN" + metac_name: "metac-o4-mini+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_TOKEN }} @@ -1223,6 +1416,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_4_1_TOKEN" + metac_name: "metac-gpt-4-1+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_4_1_TOKEN }} @@ -1237,6 +1431,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_4_1_MINI_TOKEN" + metac_name: "metac-gpt-4-1-mini+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_4_1_MINI_TOKEN }} @@ -1251,6 +1446,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_4_1_NANO_TOKEN" + metac_name: "metac-gpt-4-1-nano+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_4_1_NANO_TOKEN }} @@ -1265,6 +1461,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN" + metac_name: "metac-gemini-2-5-flash+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN }} @@ -1344,6 +1541,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_4O_TOKEN" + metac_name: "metac-gpt-4o+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_4O_TOKEN }} @@ -1358,6 +1556,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_4O_MINI_TOKEN" + metac_name: "metac-gpt-4o-mini+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_4O_MINI_TOKEN }} @@ -1372,6 +1571,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_3_5_TURBO_TOKEN" + metac_name: "metac-gpt-3-5-turbo+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_3_5_TURBO_TOKEN }} @@ -1464,6 +1664,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_LLAMA_4_MAVERICK_17B_TOKEN" + metac_name: "metac-llama-4-maverick-17b+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_LLAMA_4_MAVERICK_17B_TOKEN }} @@ -1477,6 +1678,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_QWEN_2_5_MAX_TOKEN" + metac_name: "metac-qwen-2-5-max+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN_2_5_MAX_TOKEN }} @@ -1490,6 +1692,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_R1_TOKEN" + metac_name: "metac-deepseek-r1+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_TOKEN }} @@ -1503,6 +1706,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_DEEPSEEK_V3_TOKEN" + metac_name: "metac-deepseek-v3+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_V3_TOKEN }} @@ -1516,6 +1720,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_3_LATEST_TOKEN" + metac_name: "metac-grok-3+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_LATEST_TOKEN }} @@ -1530,6 +1735,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GROK_3_MINI_LATEST_HIGH_TOKEN" + metac_name: "metac-grok-3-mini-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_MINI_LATEST_HIGH_TOKEN }} diff --git a/run_bots.py b/run_bots.py index 35ccc195..ae71edd7 100644 --- a/run_bots.py +++ b/run_bots.py @@ -125,6 +125,7 @@ class RunBotConfig(BaseModel): bot: ForecastBot | None estimated_cost_per_question: float | None tournaments: list[AllowedTourn] + metac_name: str | None = None # Metaculus username for this bot (key in METACULUS_TOKENS) model_config = {"arbitrary_types_allowed": True} @@ -562,6 +563,100 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR mode_base_bot_mapping = { # "METAC_GROK_4_1_HIGH": {} # TODO: Add these bots to github workflow. Its not yet released via API as of Dec 21st, 2025 # "METAC_GROK_4_1": {} + ############################ Bots started in April 2026 ############################ + "METAC_CLAUDE_SONNET_4_6": { + "estimated_cost_per_question": roughly_sonnet_4_cost, + "bot": create_bot( + llm=GeneralLlm( + model="anthropic/claude-sonnet-4-6", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-claude-sonnet-4-6", + }, + "METAC_QWEN_3_5": { + "estimated_cost_per_question": roughly_sonnet_3_5_cost / 2, + "bot": create_bot( + GeneralLlm( + model="openrouter/qwen/qwen3.5-397b-a17b", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-qwen-3-5", + }, + "METAC_GEMINI_3_1_PRO": { + "estimated_cost_per_question": roughly_gemini_2_5_pro_preview_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/google/gemini-3.1-pro-preview", + temperature=default_temperature, + timeout=gemini_default_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-gemini-3-1-pro", + }, + "METAC_GPT_5_5": { + "estimated_cost_per_question": roughly_gpt_5_high_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.5", + temperature=None, + timeout=gpt_5_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-gpt-5-5", + }, + "METAC_GPT_5_4": { + "estimated_cost_per_question": roughly_gpt_5_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.4", + temperature=default_temperature, + timeout=gpt_5_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-gpt-5-4", + }, + "METAC_GPT_5_4_MINI": { + "estimated_cost_per_question": roughly_gpt_4o_mini_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.4-mini", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-gpt-5-4-mini", + }, + "METAC_GPT_5_4_NANO": { + "estimated_cost_per_question": roughly_gpt_4o_mini_cost / 2, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.4-nano", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-gpt-5-4-nano", + }, + # "METAC_GPT_5_3": {} -> Not yet available in OpenAI API as of April 2026 + # "METAC_GPT_5_3_INSTANT": {} -> Not yet available in OpenAI API as of April 2026 + "METAC_MINIMAX_M2_7": { + "estimated_cost_per_question": roughly_deepseek_r1_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/minimax/minimax-m2.7", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + "metac_name": "metac-minimax-m2-7", + }, ############################ Bots started in February 2026 ############################ "METAC_CLAUDE_OPUS_4_6_HIGH_32K": { "estimated_cost_per_question": roughly_opus_4_5_cost * 1.3, From 0d666a1657cb78e00d1ab07025f319fca28c08d7 Mon Sep 17 00:00:00 2001 From: lsabor Date: Thu, 7 May 2026 14:31:45 -0700 Subject: [PATCH 03/11] add +asknews to new metac names --- .github/workflows/run-bot-aib-tournament.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 845b827e..76703a30 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -121,7 +121,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_SONNET_4_6" - metac_name: "metac-claude-sonnet-4-6" + metac_name: "metac-claude-sonnet-4-6+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -136,7 +136,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_QWEN_3_5" - metac_name: "metac-qwen-3-5" + metac_name: "metac-qwen-3-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -150,7 +150,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GEMINI_3_1_PRO" - metac_name: "metac-gemini-3-1-pro" + metac_name: "metac-gemini-3-1-pro+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -164,7 +164,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_5" - metac_name: "metac-gpt-5-5" + metac_name: "metac-gpt-5-5+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -178,7 +178,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_4" - metac_name: "metac-gpt-5-4" + metac_name: "metac-gpt-5-4+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -192,7 +192,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_4_MINI" - metac_name: "metac-gpt-5-4-mini" + metac_name: "metac-gpt-5-4-mini+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -206,7 +206,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_4_NANO" - metac_name: "metac-gpt-5-4-nano" + metac_name: "metac-gpt-5-4-nano+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -220,7 +220,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_3" - metac_name: "metac-gpt-5-3" + metac_name: "metac-gpt-5-3+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -234,7 +234,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GPT_5_3_INSTANT" - metac_name: "metac-gpt-5-3-instant" + metac_name: "metac-gpt-5-3-instant+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -248,7 +248,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_MINIMAX_M2_7" - metac_name: "metac-minimax-m2-7" + metac_name: "metac-minimax-m2-7+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} From c4301bc3c9e0c57bc8038008053e46544093c753 Mon Sep 17 00:00:00 2001 From: lsabor Date: Thu, 7 May 2026 14:34:04 -0700 Subject: [PATCH 04/11] remove metac_name from 'bot' details in run_bots --- run_bots.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/run_bots.py b/run_bots.py index ae71edd7..5073d6e5 100644 --- a/run_bots.py +++ b/run_bots.py @@ -125,7 +125,6 @@ class RunBotConfig(BaseModel): bot: ForecastBot | None estimated_cost_per_question: float | None tournaments: list[AllowedTourn] - metac_name: str | None = None # Metaculus username for this bot (key in METACULUS_TOKENS) model_config = {"arbitrary_types_allowed": True} @@ -573,7 +572,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-claude-sonnet-4-6", }, "METAC_QWEN_3_5": { "estimated_cost_per_question": roughly_sonnet_3_5_cost / 2, @@ -584,7 +582,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-qwen-3-5", }, "METAC_GEMINI_3_1_PRO": { "estimated_cost_per_question": roughly_gemini_2_5_pro_preview_cost, @@ -596,7 +593,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-gemini-3-1-pro", }, "METAC_GPT_5_5": { "estimated_cost_per_question": roughly_gpt_5_high_cost, @@ -608,7 +604,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-gpt-5-5", }, "METAC_GPT_5_4": { "estimated_cost_per_question": roughly_gpt_5_cost, @@ -620,7 +615,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-gpt-5-4", }, "METAC_GPT_5_4_MINI": { "estimated_cost_per_question": roughly_gpt_4o_mini_cost, @@ -631,7 +625,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-gpt-5-4-mini", }, "METAC_GPT_5_4_NANO": { "estimated_cost_per_question": roughly_gpt_4o_mini_cost / 2, @@ -642,7 +635,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-gpt-5-4-nano", }, # "METAC_GPT_5_3": {} -> Not yet available in OpenAI API as of April 2026 # "METAC_GPT_5_3_INSTANT": {} -> Not yet available in OpenAI API as of April 2026 @@ -655,7 +647,6 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR ), ), "tournaments": TournConfig.aib_and_site, - "metac_name": "metac-minimax-m2-7", }, ############################ Bots started in February 2026 ############################ "METAC_CLAUDE_OPUS_4_6_HIGH_32K": { From 8e3cf9e19f92fb49ad89e07e2f16147b558301c7 Mon Sep 17 00:00:00 2001 From: jerry Date: Thu, 7 May 2026 17:48:29 -0400 Subject: [PATCH 05/11] remove grok 4.1 --- .github/workflows/run-bot-aib-tournament.yaml | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 76703a30..3c712492 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -86,34 +86,6 @@ jobs: # NOTE: don't remove any of the open source models, since these are the best option for a long term baseline (other models get deprecated) - bot_grok_4_1_high: # TODO: Not yet released via API as of Dec 21st, 2025 - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4_1_HIGH" - metac_name: "metac-grok-4-1-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_HIGH }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - - bot_grok_4_1: # TODO: Not yet released via API as of Dec 21st, 2025 - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4_1" - metac_name: "metac-grok-4-1+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1 }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - #################################### April 2026 new bots #################################### bot_claude_sonnet_4_6: From eb63efbe4912398381f06511ecf26a3638126795 Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 10:09:51 -0400 Subject: [PATCH 06/11] add all new bots for summer tournament --- .github/workflows/run-bot-aib-tournament.yaml | 202 ++++++++++++++++++ run_bots.py | 158 +++++++++++++- 2 files changed, 358 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 3c712492..6fa33b2d 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -86,6 +86,208 @@ jobs: # NOTE: don't remove any of the open source models, since these are the best option for a long term baseline (other models get deprecated) + #################################### May 2026 new bots #################################### + + bot_gpt_5_5_high: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_5_HIGH" + metac_name: "metac-gpt-5-5-high+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_5_instant: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_5_INSTANT" + metac_name: "metac-gpt-5-5-instant+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gpt_5_4_high: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GPT_5_4_HIGH" + metac_name: "metac-gpt-5-4-high+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gemini_3_1_pro_high: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GEMINI_3_1_PRO_HIGH" + metac_name: "metac-gemini-3-1-pro-high+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gemini_3_1_flash_lite: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GEMINI_3_1_FLASH_LITE" + metac_name: "metac-gemini-3-1-flash-lite+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_qwen_3_6_plus: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_QWEN_3_6_PLUS" + metac_name: "metac-qwen-3-6-plus+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_gemma_4: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GEMMA_4" + metac_name: "metac-gemma-4+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_glm_5_1: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GLM_5_1" + metac_name: "metac-glm-5-1+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_claude_opus_4_7_high_32k: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_CLAUDE_OPUS_4_7_HIGH_32K" + metac_name: "metac-claude-opus-4-7-high-32k+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + + bot_kimi_k2_6: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_KIMI_K2_6" + metac_name: "metac-kimi-k2-6+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_deepseek_v4_pro_high: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_DEEPSEEK_V4_PRO_HIGH" + metac_name: "metac-deepseek-v4-pro-high+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_grok_4_3_high: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GROK_4_3_HIGH" + metac_name: "metac-grok-4-3-high+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_grok_4_20: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GROK_4_20" + metac_name: "metac-grok-4-20+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + + bot_grok_4_20_multi_agent: + needs: precache_asknews + uses: ./.github/workflows/run-bot-launcher.yaml + with: + bot_name: "METAC_GROK_4_20_MULTI_AGENT" + metac_name: "metac-grok-4-20-multi-agent+asknews" + cache_key: asknews-cache-${{ github.run_id }} + secrets: + INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + #################################### April 2026 new bots #################################### bot_claude_sonnet_4_6: diff --git a/run_bots.py b/run_bots.py index 5073d6e5..40ea6dae 100644 --- a/run_bots.py +++ b/run_bots.py @@ -560,8 +560,162 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR } mode_base_bot_mapping = { - # "METAC_GROK_4_1_HIGH": {} # TODO: Add these bots to github workflow. Its not yet released via API as of Dec 21st, 2025 - # "METAC_GROK_4_1": {} + ############################ Bots started in May 2026 ############################ + "METAC_GPT_5_5_HIGH": { + "estimated_cost_per_question": roughly_gpt_5_high_cost * 2, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.5", + reasoning_effort="high", + temperature=None, + timeout=gpt_5_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GPT_5_5_INSTANT": { + "estimated_cost_per_question": roughly_gpt_5_cost * 2, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.5", + reasoning_effort="minimal", + temperature=None, + timeout=gpt_5_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GPT_5_4_HIGH": { + "estimated_cost_per_question": roughly_gpt_5_high_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openai/gpt-5.4", + reasoning_effort="high", + temperature=default_temperature, + timeout=gpt_5_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GEMINI_3_1_PRO_HIGH": { + "estimated_cost_per_question": roughly_gemini_2_5_pro_preview_cost * 1.3, + "bot": create_bot( + GeneralLlm( + model="openrouter/google/gemini-3.1-pro-preview", + reasoning_effort="high", + temperature=default_temperature, + timeout=gemini_default_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GEMINI_3_1_FLASH_LITE": { + "estimated_cost_per_question": roughly_gpt_4o_mini_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/google/gemini-3.1-flash-lite", + temperature=default_temperature, + timeout=gemini_default_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_QWEN_3_6_PLUS": { + "estimated_cost_per_question": roughly_deepseek_r1_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/qwen/qwen3.6-plus", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GEMMA_4": { + "estimated_cost_per_question": roughly_gpt_4o_mini_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/google/gemma-4-31b-it", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GLM_5_1": { + "estimated_cost_per_question": roughly_deepseek_r1_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openrouter/z-ai/glm-5.1", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_CLAUDE_OPUS_4_7_HIGH_32K": { + "estimated_cost_per_question": roughly_opus_4_5_cost * 1.5, + "bot": create_bot( + llm=GeneralLlm( + model="anthropic/claude-opus-4-7", + **claude_thinking_settings_32k, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_KIMI_K2_6": { + "estimated_cost_per_question": roughly_deepseek_r1_cost, + "bot": create_bot( + GeneralLlm( + model="openrouter/moonshotai/kimi-k2.6", + temperature=default_temperature, + timeout=kimi_k2_timeout, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_DEEPSEEK_V4_PRO_HIGH": { + "estimated_cost_per_question": roughly_deepseek_r1_cost, + "bot": create_bot( + llm=GeneralLlm( + model="openrouter/deepseek/deepseek-v4-pro", + temperature=default_temperature, + reasoning={ + "effort": "high", + }, + timeout=5 * 60, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GROK_4_3_HIGH": { + "estimated_cost_per_question": 5 * roughly_one_call_to_grok_4_llm, + "bot": create_bot( + llm=GeneralLlm( + model="openrouter/x-ai/grok-4.3", + reasoning_effort="high", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GROK_4_20": { + "estimated_cost_per_question": 5 * roughly_one_call_to_grok_4_llm, + "bot": create_bot( + llm=GeneralLlm( + model="openrouter/x-ai/grok-4.20", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, + "METAC_GROK_4_20_MULTI_AGENT": { + "estimated_cost_per_question": 10 * roughly_one_call_to_grok_4_llm, + "bot": create_bot( + llm=GeneralLlm( + model="openrouter/x-ai/grok-4.20-multi-agent", + temperature=default_temperature, + ), + ), + "tournaments": TournConfig.aib_and_site, + }, ############################ Bots started in April 2026 ############################ "METAC_CLAUDE_SONNET_4_6": { "estimated_cost_per_question": roughly_sonnet_4_cost, From 206e0dd0275254f4bcbb93d791206482e4f112f5 Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 10:23:00 -0400 Subject: [PATCH 07/11] remove old bots --- .github/workflows/run-bot-aib-tournament.yaml | 574 +++++++++--------- run_bots.py | 38 +- 2 files changed, 306 insertions(+), 306 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 6fa33b2d..5114811a 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -347,19 +347,19 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gpt_5_4: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_4" - metac_name: "metac-gpt-5-4+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gpt_5_4: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_4" + # metac_name: "metac-gpt-5-4+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} bot_gpt_5_4_mini: needs: precache_asknews @@ -389,33 +389,33 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gpt_5_3: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_3" - metac_name: "metac-gpt-5-3+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gpt_5_3: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_3" + # metac_name: "metac-gpt-5-3+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gpt_5_3_instant: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_3_INSTANT" - metac_name: "metac-gpt-5-3-instant+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gpt_5_3_instant: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_3_INSTANT" + # metac_name: "metac-gpt-5-3-instant+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} bot_minimax_m2_7: needs: precache_asknews @@ -534,65 +534,65 @@ jobs: #################################### December 2025 new bots #################################### - bot_claude_opus_4_5_high_32k: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_CLAUDE_OPUS_4_5_HIGH_32K" - metac_name: "metac-claude-opus-4-5-high-32k+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5_HIGH_32K }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_claude_opus_4_5_high_32k: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_CLAUDE_OPUS_4_5_HIGH_32K" + # metac_name: "metac-claude-opus-4-5-high-32k+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5_HIGH_32K }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_claude_opus_4_5: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_CLAUDE_OPUS_4_5" - metac_name: "metac-claude-opus-4-5+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_claude_opus_4_5: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_CLAUDE_OPUS_4_5" + # metac_name: "metac-claude-opus-4-5+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_5 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gpt_5_2_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_2_HIGH" - metac_name: "metac-gpt-5-2-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_2_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_2_HIGH" + # metac_name: "metac-gpt-5-2-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gpt_5_2: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_2" - metac_name: "metac-gpt-5-2+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_2: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_2" + # metac_name: "metac-gpt-5-2+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_2 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # bot_llama_3_1_nemotron_ultra_253b: # needs: precache_asknews @@ -607,20 +607,20 @@ jobs: # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gemini_3_flash: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GEMINI_3_FLASH" - metac_name: "metac-gemini-3-flash+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_3_FLASH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gemini_3_flash: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GEMINI_3_FLASH" + # metac_name: "metac-gemini-3-flash+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_3_FLASH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} bot_glm_4_6: needs: precache_asknews @@ -664,35 +664,35 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gpt_5_1_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_1_HIGH" - metac_name: "metac-gpt-5-1-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_1_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_1_HIGH" + # metac_name: "metac-gpt-5-1-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gpt_5_1: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_1" - metac_name: "metac-gpt-5-1+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_1: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_1" + # metac_name: "metac-gpt-5-1+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_1 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # bot_gemini_3_pro_high: # needs: precache_asknews @@ -777,35 +777,35 @@ jobs: #################################### October 2025 new bots #################################### - bot_claude_4_5_sonnet_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_CLAUDE_4_5_SONNET_HIGH" - metac_name: "metac-claude-4-5-sonnet-high-32k+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_claude_4_5_sonnet_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_CLAUDE_4_5_SONNET_HIGH" + # metac_name: "metac-claude-4-5-sonnet-high-32k+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_claude_4_5_sonnet: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_CLAUDE_4_5_SONNET" - metac_name: "metac-claude-4-5-sonnet+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_claude_4_5_sonnet: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_CLAUDE_4_5_SONNET" + # metac_name: "metac-claude-4-5-sonnet+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_4_5_SONNET }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} bot_qwen_3_max: needs: precache_asknews @@ -847,20 +847,20 @@ jobs: # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_grok_4_fast_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4_FAST_HIGH" - metac_name: "metac-grok-4-fast-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_FAST_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_grok_4_fast_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_4_FAST_HIGH" + # metac_name: "metac-grok-4-fast-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_FAST_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_grok_4_fast: # needs: precache_asknews @@ -904,35 +904,35 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gpt_5_mini: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_MINI" - metac_name: "metac-gpt-5-mini+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_MINI }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_mini: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_MINI" + # metac_name: "metac-gpt-5-mini+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_MINI }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gpt_5_nano: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_NANO" - metac_name: "metac-gpt-5-nano+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_NANO }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gpt_5_nano: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_NANO" + # metac_name: "metac-gpt-5-nano+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GPT_5_NANO }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # bot_claude_4_sonnet_high_16k: # needs: precache_asknews @@ -973,20 +973,20 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_grok_4: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4" - metac_name: "metac-grok-4+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_grok_4: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_4" + # metac_name: "metac-grok-4+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} bot_kimi_k2: needs: precache_asknews @@ -1158,18 +1158,18 @@ jobs: # INPUT_EXA_API_KEY: ${{ secrets.EXA_API_KEY }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gemini_2_5_pro_grounding: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GEMINI_2_5_PRO_GROUNDING" - metac_name: "metac-gemini-2-5-pro-grounding[research-only]" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_PRO_GROUNDING }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_gemini_2_5_pro_grounding: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GEMINI_2_5_PRO_GROUNDING" + # metac_name: "metac-gemini-2-5-pro-grounding[research-only]" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_PRO_GROUNDING }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} bot_asknews_deepnews: needs: precache_asknews @@ -1477,18 +1477,18 @@ jobs: INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - bot_deepseek_r1_gemini_2_5_pro_grounding: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING" - metac_name: "metac-deepseek-r1+gemini-2-5-pro-grounding" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # bot_deepseek_r1_gemini_2_5_pro_grounding: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING" + # metac_name: "metac-deepseek-r1+gemini-2-5-pro-grounding" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} # bot_deepseek_r1_exa_smart_searcher: # needs: precache_asknews @@ -1540,20 +1540,20 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_o3: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_O3_TOKEN" - metac_name: "metac-o3+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O3_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_o3: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_O3_TOKEN" + # metac_name: "metac-o3+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O3_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} bot_o4_mini_high: needs: precache_asknews @@ -1630,20 +1630,20 @@ jobs: INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_gemini_2_5_flash: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN" - metac_name: "metac-gemini-2-5-flash+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gemini_2_5_flash: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN" + # metac_name: "metac-gemini-2-5-flash+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GEMINI_2_5_FLASH_PREVIEW_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_o1_high: # needs: precache_asknews @@ -1904,20 +1904,20 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_grok_3_mini_latest_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_3_MINI_LATEST_HIGH_TOKEN" - metac_name: "metac-grok-3-mini-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_MINI_LATEST_HIGH_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_grok_3_mini_latest_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_3_MINI_LATEST_HIGH_TOKEN" + # metac_name: "metac-grok-3-mini-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_MINI_LATEST_HIGH_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} ### Special bots bot_uniform_probability: diff --git a/run_bots.py b/run_bots.py index 40ea6dae..2316d36c 100644 --- a/run_bots.py +++ b/run_bots.py @@ -768,7 +768,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gpt_5_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_4_MINI": { "estimated_cost_per_question": roughly_gpt_4o_mini_cost, @@ -887,7 +887,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR **claude_thinking_settings_32k, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_CLAUDE_OPUS_4_5": { "estimated_cost_per_question": roughly_opus_4_5_cost, @@ -897,7 +897,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_only, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_2_HIGH": { "estimated_cost_per_question": roughly_gpt_5_high_cost * 1.5, @@ -909,7 +909,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gpt_5_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_2": { "estimated_cost_per_question": roughly_gpt_5_cost * 1.5, @@ -920,7 +920,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gpt_5_timeout, ), ), - "tournaments": TournConfig.aib_only, + "tournaments": TournConfig.NONE, }, "METAC_LLAMA_3_1_NEMOTRON_ULTRA_253B": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -941,7 +941,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gemini_default_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GLM_4_6": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -986,7 +986,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_1": { "estimated_cost_per_question": roughly_gpt_5_cost, @@ -998,7 +998,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, # "METAC_GEMINI_3_PRO_HIGH": {} # The default for regular gemini 3 pro is "high" so no need to make a separate version # "METAC_GEMINI_3_PRO": { @@ -1044,7 +1044,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR **claude_thinking_settings_32k, ), ), - "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], + "tournaments": TournConfig.NONE, }, "METAC_CLAUDE_4_5_SONNET": { "estimated_cost_per_question": roughly_sonnet_4_cost, @@ -1054,7 +1054,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_QWEN_3_MAX": { "estimated_cost_per_question": roughly_sonnet_3_5_cost / 2, @@ -1102,7 +1102,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GROK_4_FAST": { "estimated_cost_per_question": guess_at_deepseek_v3_1_cost, @@ -1150,7 +1150,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_NANO": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -1160,7 +1160,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_CLAUDE_4_SONNET_HIGH_16K": { "estimated_cost_per_question": 0.33980, @@ -1200,7 +1200,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], + "tournaments": TournConfig.NONE, }, "METAC_KIMI_K2": { **kimi_k2_basic_bot, @@ -1313,7 +1313,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR gemini_grounding_llm, bot_type="research_only", ), - "tournaments": TournConfig.aib_only, + "tournaments": TournConfig.NONE, }, "METAC_ASKNEWS_DEEPNEWS": { "estimated_cost_per_question": 0, @@ -1563,7 +1563,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR default_research_comparison_forecast_llm, researcher=gemini_grounding_llm, ), - "tournaments": TournConfig.aib_only, + "tournaments": TournConfig.NONE, }, "METAC_DEEPSEEK_R1_EXA_SMART_SEARCHER": { "estimated_cost_per_question": guess_at_deepseek_plus_search, @@ -1615,7 +1615,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], + "tournaments": TournConfig.NONE, }, "METAC_O4_MINI_HIGH_TOKEN": { "estimated_cost_per_question": 0.07, @@ -1677,7 +1677,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gemini_default_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_O1_HIGH_TOKEN": { "estimated_cost_per_question": 1.18, @@ -1883,7 +1883,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR reasoning_effort="high", ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_UNIFORM_PROBABILITY_BOT_TOKEN": { "estimated_cost_per_question": 0.00, From d94fc8d07503a853864181b0bf9cab7326116450 Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 12:20:20 -0400 Subject: [PATCH 08/11] add a few bots to Metaculus Cup --- run_bots.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run_bots.py b/run_bots.py index 2316d36c..8692e91c 100644 --- a/run_bots.py +++ b/run_bots.py @@ -571,7 +571,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gpt_5_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], }, "METAC_GPT_5_5_INSTANT": { "estimated_cost_per_question": roughly_gpt_5_cost * 2, @@ -607,7 +607,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gemini_default_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], }, "METAC_GEMINI_3_1_FLASH_LITE": { "estimated_cost_per_question": roughly_gpt_4o_mini_cost, @@ -658,7 +658,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR **claude_thinking_settings_32k, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], }, "METAC_KIMI_K2_6": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -1127,7 +1127,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.NONE, + "tournaments": [AllowedTourn.METACULUS_CUP], }, "METAC_GPT_5": { "estimated_cost_per_question": roughly_gpt_5_cost, From eed7f88a6a8affa9e3ad1dda3820291d2ea5179c Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 14:25:00 -0400 Subject: [PATCH 09/11] minor name mismatches and addition of missing OAI API key --- .github/workflows/run-bot-aib-tournament.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 5114811a..167f37d0 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -99,6 +99,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} @@ -113,6 +114,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} @@ -127,6 +129,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} @@ -177,7 +180,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_GEMMA_4" - metac_name: "metac-gemma-4+asknews" + metac_name: "metac-gemma-4-26b+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -205,7 +208,7 @@ jobs: uses: ./.github/workflows/run-bot-launcher.yaml with: bot_name: "METAC_CLAUDE_OPUS_4_7_HIGH_32K" - metac_name: "metac-claude-opus-4-7-high-32k+asknews" + metac_name: "metac-claude-opus-4-7-high+asknews" cache_key: asknews-cache-${{ github.run_id }} secrets: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} @@ -344,6 +347,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} @@ -372,6 +376,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} @@ -386,6 +391,7 @@ jobs: INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} From b531bc43d61b60b2c9a72b2e51171d95f79b4e55 Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 14:46:47 -0400 Subject: [PATCH 10/11] Opus 4.6 -> Opus 4.7 migration for thinking settings --- run_bots.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/run_bots.py b/run_bots.py index 8692e91c..8cf39235 100644 --- a/run_bots.py +++ b/run_bots.py @@ -412,6 +412,18 @@ def make_claude_thinking_settings(thinking_tokens: int, max_tokens: int) -> dict } +def make_claude_adaptive_thinking_settings( + effort: Literal["low", "medium", "high"], max_tokens: int +) -> dict: + return { + "temperature": 1, + "thinking": {"type": "adaptive"}, + "extra_body": {"output_config": {"effort": effort}}, + "max_tokens": max_tokens, + "timeout": 160, + } + + def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR """ Each entry in the dict has a key which is the environment variable set in the project secrets, and also used in the Workflows that run the bots. @@ -466,6 +478,9 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR claude_thinking_settings_32k: dict = make_claude_thinking_settings( thinking_tokens=32000, max_tokens=64000 ) + claude_adaptive_thinking_settings_high: dict = ( + make_claude_adaptive_thinking_settings(effort="high", max_tokens=64000) + ) gpt_5_timeout = 15 * 60 gemini_grounding_llm = GeneralLlm( @@ -655,7 +670,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR "bot": create_bot( llm=GeneralLlm( model="anthropic/claude-opus-4-7", - **claude_thinking_settings_32k, + **claude_adaptive_thinking_settings_high, ), ), "tournaments": TournConfig.aib_and_site + [AllowedTourn.METACULUS_CUP], From 0d34fad1829aa84f74e6c57a66272ea3129429fb Mon Sep 17 00:00:00 2001 From: jerry Date: Tue, 12 May 2026 15:04:25 -0400 Subject: [PATCH 11/11] effort level for Opus 4.7 specified in output_config instead of extra_body --- run_bots.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_bots.py b/run_bots.py index 8cf39235..50b12bdd 100644 --- a/run_bots.py +++ b/run_bots.py @@ -418,7 +418,8 @@ def make_claude_adaptive_thinking_settings( return { "temperature": 1, "thinking": {"type": "adaptive"}, - "extra_body": {"output_config": {"effort": effort}}, + "output_config": {"effort": effort}, + "allowed_openai_params": ["output_config"], "max_tokens": max_tokens, "timeout": 160, }