diff --git a/kola-denylist.yaml b/kola-denylist.yaml
index 20162a90..55e182bb 100644
--- a/kola-denylist.yaml
+++ b/kola-denylist.yaml
@@ -2,9 +2,6 @@
 # coreos-assembler to automatically skip some tests. For more information,
 # see: https://github.com/coreos/coreos-assembler/pull/866.
 
-- pattern: ext.config.version.rhaos-pkgs-match-openshift
-  tracker: https://github.com/openshift/os/issues/1847
-
 - pattern: rhcos.network.init-interfaces-test
   tracker: https://github.com/openshift/os/issues/1852
   arches:
diff --git a/tests/kola/files/openvswitch-hugetlbfs-groups b/tests/kola/files/openvswitch-hugetlbfs-groups
index 0251e3ad..7f939048 100755
--- a/tests/kola/files/openvswitch-hugetlbfs-groups
+++ b/tests/kola/files/openvswitch-hugetlbfs-groups
@@ -3,6 +3,7 @@
 ##   exclusive: false
 ##   architectures: "x86_64 ppc64le"
 ##   description: Verify openvswitch user is in the hugetlbfs group.
+##   tags: openshift
 
 set -xeuo pipefail
 
diff --git a/tests/kola/version/rhaos-pkgs-match-openshift b/tests/kola/version/rhaos-pkgs-match-openshift
index 07a7ecdf..ff47408d 100755
--- a/tests/kola/version/rhaos-pkgs-match-openshift
+++ b/tests/kola/version/rhaos-pkgs-match-openshift
@@ -4,6 +4,7 @@
 ##   description: Verify that packages coming from the rhaos
 ##     branches match the OpenShift version.
 ##     This is RHCOS only.
+##   tags: openshift
 
 set -xeuo pipefail
 
@@ -13,7 +14,21 @@ set -xeuo pipefail
 # source the environment variables to get OPENSHIFT_VERSION
 source /etc/os-release
 
-rpm -qa | grep "rhaos" > /tmp/rhaos-packages.txt
+# These are packages that are known to not match the
+# targeted version of openshift. Exlude them from the test.
+RHAOS_PACKAGE_EXCEPTIONS=(
+    # toolbox doesn't always match the target ocp version because it
+    # is included in the base-image which spans multiple versions
+    toolbox
+    # conmon currently is missing 4.19+ vesions of the package
+    conmon-rs
+)
+
+exception_pattern=$(echo ${RHAOS_PACKAGE_EXCEPTIONS[@]} | tr " " "|")
+
+# Use `|| true` here to allow situations where RHAOS_PACKAGE_EXCEPTIONS contains every
+# rhaos package in the image, which would otherwise cause `grep -Ev` to fail the test.
+rpm -qa | grep "rhaos" | grep -Ev "${exception_pattern}" > /tmp/rhaos-packages.txt || true
 
 if grep -v "rhaos${OPENSHIFT_VERSION}" /tmp/rhaos-packages.txt; then
     fatal "Error: rhaos packages do not match OpenShift version"