Added verify_help_text() helper function for tests and removed BASE_HELP and BASE_HELP_VERBOSE constants

The tests are now much more resilient to adding, removing, or renaming commands

Author: tleonhardt

tests/conftest.py

@@ -4,7 +4,7 @@
 """
 import sys
 from contextlib import redirect_stdout, redirect_stderr
-from typing import Optional
+from typing import List, Optional, Union
 from unittest import mock
 
 from pytest import fixture
@@ -25,31 +25,23 @@
     except ImportError:
         pass
 
-# Help text for base cmd2.Cmd application
-BASE_HELP = """Documented commands (type help <topic>):
-========================================
-alias  help     load   py        quit          run_script  shell    
-edit   history  macro  pyscript  run_pyscript  set         shortcuts
-"""  # noqa: W291
-
-BASE_HELP_VERBOSE = """
-Documented commands (type help <topic>):
-================================================================================
-alias               Manage aliases
-edit                Edit a file in a text editor
-help                List available commands or provide detailed help for a specific command
-history             View, run, edit, save, or clear previously entered commands
-load                Run commands in script file that is encoded as either ASCII or UTF-8 text
-macro               Manage macros
-py                  Invoke Python command or shell
-pyscript            Run a Python script file inside the console
-quit                Exit this application
-run_pyscript        Run a Python script file inside the console
-run_script          Run commands in script file that is encoded as either ASCII or UTF-8 text
-set                 Set a settable parameter or show current settings of parameters
-shell               Execute a command as if at the OS prompt
-shortcuts           List available shortcuts
-"""
+
+def verify_help_text(cmd2_app: cmd2.Cmd, help_output: Union[str, List[str]]) -> None:
+    """This function verifies that all expected commands are present in the help text.
+
+    :param cmd2_app: instance of cmd2.Cmd
+    :param help_output: output of help, either as a string or list of strings
+    """
+    if isinstance(help_output, str):
+        help_text = help_output
+    else:
+        help_text = ''.join(help_output)
+    commands = cmd2_app.get_visible_commands()
+    for command in commands:
+        assert command in help_text
+
+    # TODO: Consider adding checks for categories and for verbose history
+
 
 # Help text for the history command
 HELP_HISTORY = """Usage: history [-h] [-r | -e | -o FILE | -t TRANSCRIPT | -c] [-s] [-x] [-v]

tests/test_cmd2.py

@@ -22,8 +22,7 @@
 
 import cmd2
 from cmd2 import clipboard, constants, utils
-from .conftest import run_cmd, normalize, BASE_HELP, BASE_HELP_VERBOSE, \
-    HELP_HISTORY, SHORTCUTS_TXT, SHOW_TXT, SHOW_LONG
+from .conftest import run_cmd, normalize, verify_help_text, HELP_HISTORY, SHORTCUTS_TXT, SHOW_TXT, SHOW_LONG
 
 def CreateOutsimApp():
     c = cmd2.Cmd()
@@ -53,21 +52,20 @@ def test_empty_statement(base_app):
 
 def test_base_help(base_app):
     out, err = run_cmd(base_app, 'help')
-    expected = normalize(BASE_HELP)
-    assert out == expected
+    verify_help_text(base_app, out)
 
 def test_base_help_verbose(base_app):
     out, err = run_cmd(base_app, 'help -v')
-    expected = normalize(BASE_HELP_VERBOSE)
-    assert out == expected
+    verify_help_text(base_app, out)
 
     # Make sure :param type lines are filtered out of help summary
     help_doc = base_app.do_help.__func__.__doc__
     help_doc += "\n:param fake param"
     base_app.do_help.__func__.__doc__ = help_doc
 
     out, err = run_cmd(base_app, 'help --verbose')
-    assert out == expected
+    verify_help_text(base_app, out)
+    assert ':param' not in ''.join(out)
 
 def test_base_argparse_help(base_app):
     # Verify that "set -h" gives the same output as "help set" and that it starts in a way that makes sense
@@ -427,18 +425,17 @@ def test_output_redirection(base_app):
     try:
         # Verify that writing to a file works
         run_cmd(base_app, 'help > {}'.format(filename))
-        expected = normalize(BASE_HELP)
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
+            content = f.read()
+        verify_help_text(base_app, content)
 
         # Verify that appending to a file also works
         run_cmd(base_app, 'help history >> {}'.format(filename))
-        expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
-    except:
+            appended_content = f.read()
+        assert appended_content.startswith(content)
+        assert len(appended_content) > len(content)
+    except Exception:
         raise
     finally:
         os.remove(filename)
@@ -448,19 +445,18 @@ def test_output_redirection_to_nonexistent_directory(base_app):
 
     # Verify that writing to a file in a non-existent directory doesn't work
     run_cmd(base_app, 'help > {}'.format(filename))
-    expected = normalize(BASE_HELP)
     with pytest.raises(FileNotFoundError):
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
+            content = f.read()
+        verify_help_text(base_app, content)
 
     # Verify that appending to a file also works
     run_cmd(base_app, 'help history >> {}'.format(filename))
-    expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
     with pytest.raises(FileNotFoundError):
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
+            appended_content = f.read()
+        verify_help_text(base_app, appended_content)
+        assert len(appended_content) > len(content)
 
 def test_output_redirection_to_too_long_filename(base_app):
     filename = '~/sdkfhksdjfhkjdshfkjsdhfkjsdhfkjdshfkjdshfkjshdfkhdsfkjhewfuihewiufhweiufhiweufhiuewhiuewhfiuwehfia' \
@@ -471,19 +467,18 @@ def test_output_redirection_to_too_long_filename(base_app):
 
     # Verify that writing to a file in a non-existent directory doesn't work
     run_cmd(base_app, 'help > {}'.format(filename))
-    expected = normalize(BASE_HELP)
     with pytest.raises(OSError):
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
+            content = f.read()
+        verify_help_text(base_app, content)
 
     # Verify that appending to a file also works
     run_cmd(base_app, 'help history >> {}'.format(filename))
-    expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
     with pytest.raises(OSError):
         with open(filename) as f:
-            content = normalize(f.read())
-        assert content == expected
+            appended_content = f.read()
+        verify_help_text(base_app, content)
+        assert len(appended_content) > len(content)
 
 
 def test_feedback_to_output_true(base_app):
@@ -530,8 +525,7 @@ def test_disallow_redirection(base_app):
 
     # Verify output wasn't redirected
     out, err = run_cmd(base_app, 'help > {}'.format(filename))
-    expected = normalize(BASE_HELP)
-    assert out == expected
+    verify_help_text(base_app, out)
 
     # Verify that no file got created
     assert not os.path.exists(filename)
@@ -574,13 +568,14 @@ def test_pipe_to_shell_error(base_app):
 def test_send_to_paste_buffer(base_app):
     # Test writing to the PasteBuffer/Clipboard
     run_cmd(base_app, 'help >')
-    expected = normalize(BASE_HELP)
-    assert normalize(cmd2.cmd2.get_paste_buffer()) == expected
+    paste_contents = cmd2.cmd2.get_paste_buffer()
+    verify_help_text(base_app, paste_contents)
 
     # Test appending to the PasteBuffer/Clipboard
     run_cmd(base_app, 'help history >>')
-    expected = normalize(BASE_HELP + '\n' + HELP_HISTORY)
-    assert normalize(cmd2.cmd2.get_paste_buffer()) == expected
+    appended_contents = cmd2.cmd2.get_paste_buffer()
+    assert appended_contents.startswith(paste_contents)
+    assert len(appended_contents) > len(paste_contents)
 
 
 def test_base_timing(base_app):
@@ -901,17 +896,7 @@ def test_custom_command_help(help_app):
 
 def test_custom_help_menu(help_app):
     out, err = run_cmd(help_app, 'help')
-    expected = normalize("""
-Documented commands (type help <topic>):
-========================================
-alias  help     load   py        quit          run_script  shell      squat
-edit   history  macro  pyscript  run_pyscript  set         shortcuts
-
-Undocumented commands:
-======================
-undoc
-""")
-    assert out == expected
+    verify_help_text(help_app, out)
 
 def test_help_undocumented(help_app):
     out, err = run_cmd(help_app, 'help undoc')
@@ -962,62 +947,11 @@ def helpcat_app():
 
 def test_help_cat_base(helpcat_app):
     out, err = run_cmd(helpcat_app, 'help')
-    expected = normalize("""Documented commands (type help <topic>):
-
-Custom Category
-===============
-edit  squat
-
-Some Category
-=============
-cat_nodoc  diddly
-
-Other
-=====
-alias  history  macro  pyscript  run_pyscript  set    shortcuts
-help   load     py     quit      run_script    shell
-
-Undocumented commands:
-======================
-undoc
-""")
-    assert out == expected
+    verify_help_text(helpcat_app, out)
 
 def test_help_cat_verbose(helpcat_app):
     out, err = run_cmd(helpcat_app, 'help --verbose')
-    expected = normalize("""Documented commands (type help <topic>):
-
-Custom Category
-================================================================================
-edit                This overrides the edit command and does nothing.
-squat               This command does diddly squat...
-
-Some Category
-================================================================================
-cat_nodoc
-diddly              This command does diddly
-
-Other
-================================================================================
-alias               Manage aliases
-help                List available commands or provide detailed help for a specific command
-history             View, run, edit, save, or clear previously entered commands
-load                Run commands in script file that is encoded as either ASCII or UTF-8 text
-macro               Manage macros
-py                  Invoke Python command or shell
-pyscript            Run a Python script file inside the console
-quit                Exit this application
-run_pyscript        Run a Python script file inside the console
-run_script          Run commands in script file that is encoded as either ASCII or UTF-8 text
-set                 Set a settable parameter or show current settings of parameters
-shell               Execute a command as if at the OS prompt
-shortcuts           List available shortcuts
-
-Undocumented commands:
-======================
-undoc
-""")
-    assert out == expected
+    verify_help_text(helpcat_app, out)
 
 
 class SelectApp(cmd2.Cmd):
@@ -1656,12 +1590,10 @@ def test_multiple_aliases(base_app):
     run_cmd(base_app, 'alias create {} help'.format(alias1))
     run_cmd(base_app, 'alias create {} help -v'.format(alias2))
     out, err = run_cmd(base_app, alias1)
-    expected = normalize(BASE_HELP)
-    assert out == expected
+    verify_help_text(base_app, out)
 
     out, err = run_cmd(base_app, alias2)
-    expected = normalize(BASE_HELP_VERBOSE)
-    assert out == expected
+    verify_help_text(base_app, out)
 
 def test_macro_no_subcommand(base_app):
     out, err = run_cmd(base_app, 'macro')
@@ -1716,8 +1648,7 @@ def test_macro_create_with_args(base_app):
 
     # Run the macro
     out, err = run_cmd(base_app, 'fake help -v')
-    expected = normalize(BASE_HELP_VERBOSE)
-    assert out == expected
+    verify_help_text(base_app, out)
 
 def test_macro_create_with_escaped_args(base_app):
     # Create the macro
@@ -1810,12 +1741,11 @@ def test_multiple_macros(base_app):
     run_cmd(base_app, 'macro create {} help'.format(macro1))
     run_cmd(base_app, 'macro create {} help -v'.format(macro2))
     out, err = run_cmd(base_app, macro1)
-    expected = normalize(BASE_HELP)
-    assert out == expected
+    verify_help_text(base_app, out)
 
-    out, err = run_cmd(base_app, macro2)
-    expected = normalize(BASE_HELP_VERBOSE)
-    assert out == expected
+    out2, err2 = run_cmd(base_app, macro2)
+    verify_help_text(base_app, out2)
+    assert len(out2) > len(out)
 
 def test_nonexistent_macro(base_app):
     from cmd2.parsing import StatementParser
@@ -1878,7 +1808,7 @@ def test_onecmd_raw_str_continue(outsim_app):
     stop = outsim_app.onecmd(line)
     out = outsim_app.stdout.getvalue()
     assert not stop
-    assert normalize(out) == normalize(BASE_HELP)
+    verify_help_text(outsim_app, out)
 
 def test_onecmd_raw_str_quit(outsim_app):
     line = "quit"

tests/test_transcript.py

@@ -14,7 +14,7 @@
 import pytest
 
 import cmd2
-from .conftest import run_cmd, BASE_HELP_VERBOSE
+from .conftest import run_cmd, verify_help_text
 from cmd2 import transcript
 from cmd2.utils import StdSim
 
@@ -211,9 +211,8 @@ def test_run_script_record_transcript(base_app, request):
     with open(transcript_fname) as f:
         xscript = f.read()
 
-    expected = '(Cmd) help -v\n' + BASE_HELP_VERBOSE + '\n'
-
-    assert xscript == expected
+    assert xscript.startswith('(Cmd) help -v\n')
+    verify_help_text(base_app, xscript)
 
 
 def test_generate_transcript_stop(capsys):