test.py

#!/usr/bin/python3

import argparse
import logging
import os
from pathlib import Path
import platform
from pprint import pprint
import shutil
import socket
import subprocess
from subprocess import PIPE
import sys
from tempfile import TemporaryDirectory
from traceback import print_exc

from module.args import parse_args
from module.path import ProjectPaths
from module.profile import BranchProfile, resolve_profile
from module.util import XMAKE_ARCH_MAP, ensure

def clean(config: argparse.Namespace, paths: ProjectPaths):
  if paths.test_dir.exists():
    shutil.rmtree(paths.test_dir)

def prepare_dirs(paths: ProjectPaths):
  shutil.copytree(
    paths.test_src_dir,
    paths.test_dir,
    ignore = shutil.ignore_patterns(
      '.cache',
      '.vscode',
      '.xmake',
      'build',
    ),
  )

def extract(path: Path, arx: Path):
  if platform.system() == 'Windows':
    # Windows `tar.exe` does not support Unicode path
    # (WTF? it's a system component distributed by Microsoft!)
    with TemporaryDirectory() as tmp:
      subprocess.run([
        'bsdtar',
        '-C', tmp,
        '-xf', arx,
        '--no-same-owner',
        # workaround zstd pipe error with Windows `tar.exe`
        # https://github.com/libarchive/libarchive/issues/2512
        '--ignore-zeros',
      ], check = True)
      shutil.copytree(tmp, path, dirs_exist_ok = True)
  else:
    subprocess.run([
      'bsdtar',
      '-C', path,
      '-xf', arx,
      '--no-same-owner',
    ], check = True)

def prepare_test_binary(ver: BranchProfile, paths: ProjectPaths):
  extract(paths.test_dir, paths.test_driver_pkg)
  extract(paths.test_dir, paths.mingw_pkg)
  extract(paths.test_dir, paths.xmake_pkg)

def available_port():
  with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
    s.bind(('localhost', 0))
    return s.getsockname()[1]

def test_mingw_compiler(ver: BranchProfile, paths: ProjectPaths, verbose: list[str]):
  rel_mingw_dir = paths.test_mingw_dir.relative_to(paths.test_dir)
  xmake = paths.test_mingw_dir / 'bin/xmake.exe'
  subprocess.check_call([
    xmake, 'f', *verbose,
    '-p', 'mingw', '-a', XMAKE_ARCH_MAP[ver.arch],
    f'--mingw={rel_mingw_dir}',
  ], cwd = paths.test_dir)
  subprocess.check_call([xmake, 'b', *verbose], cwd = paths.test_dir)
  subprocess.check_call([xmake, 'test', *verbose], cwd = paths.test_dir)

def test_mingw_shared(ver: BranchProfile, paths: ProjectPaths, verbose: list[str]):
  shutil.copytree(paths.test_mingw_dir / 'lib/shared', paths.test_mingw_dir, dirs_exist_ok = True)

  rel_mingw_dir = paths.test_mingw_dir.relative_to(paths.test_dir)
  xmake = paths.test_mingw_dir / 'bin/xmake.exe'
  subprocess.check_call([
    xmake, 'f', *verbose,
    f'--builddir=build-shared',
    '-p', 'mingw', '-a', XMAKE_ARCH_MAP[ver.arch],
    f'--mingw={rel_mingw_dir}',
  ], cwd = paths.test_dir)
  subprocess.check_call([xmake, 'b', *verbose], cwd = paths.test_dir)
  subprocess.check_call([xmake, 'test', *verbose], cwd = paths.test_dir)

def test_mingw_make_gdb(ver: BranchProfile, paths: ProjectPaths):
  xmake_arch = XMAKE_ARCH_MAP[ver.arch]
  build_dir = f'build/mingw/{xmake_arch}/debug'
  inferior = f'{build_dir}/breakpoint.exe'
  ensure(paths.test_dir / build_dir)

  # make
  subprocess.check_call([
    paths.test_dir / 'set-path.exe',
    'mingw32-make.exe', f'DIR={build_dir}', 'SUFFIX=.exe',
  ], cwd = paths.test_dir)

  # gdb
  port = available_port()
  comm = f'localhost:{port}'

  gdb_command_file = 'gdb_command.txt'
  with open(paths.test_dir / gdb_command_file, 'wb') as f:
    content = (
      f'file {inferior}\n'
      'set sysroot C:\n'
      f'target remote {comm}\n'
      'b 14\n'
      'b 19\n'
      'c\n'
      'p fib[i]\n'  # i = 2, fib[i] = 1
      'c\n'
      'p fib[i]\n'  # i = 3, fib[i] = 2
      'c\n'
      'p fib[i]\n'  # i = 4, fib[i] = 3
      'c\n'
      'p fib[i]\n'  # i = 5, fib[i] = 5
      'c\n'
      'p fib_vec\n'
      'c\n'
    )
    f.write(content.encode())

  expected_output = [
    '$1 = 1',
    '$2 = 2',
    '$3 = 3',
    '$4 = 5',
    '$5 = std::vector of length 6, capacity', '= {0, 1, 1, 2, 3, 5}',
  ]

  bin_dir = paths.test_mingw_dir / 'bin'
  gdb_exe = bin_dir / 'gdb.exe'
  gdbserver_exe = bin_dir / 'gdbserver.exe'

  gdbserver = subprocess.Popen([gdbserver_exe, comm, inferior], cwd = paths.test_dir)
  gdb = subprocess.Popen([gdb_exe, '--batch', f'--command={gdb_command_file}'], cwd = paths.test_dir, stdout = PIPE)
  gdb.wait(timeout = 10.0)
  if gdb.returncode != 0:
    raise Exception(f"gdb exited with code {gdb.returncode}")
  gdbserver.wait(timeout = 1.0)
  if gdbserver.returncode != 0:
    raise Exception(f"gdbserver exited with code {gdbserver.returncode}")

  gdb_output = gdb.stdout.read().decode()
  for line in expected_output:
    if line not in gdb_output:
      raise Exception(f"expected output line '{line}' not found in gdb output:\n{gdb_output}")

def main():
  # We want UTF-8 pipe in CI (console not affected)
  if platform.system() == 'Windows':
    sys.stdout.reconfigure(encoding = 'UTF-8')

  config = parse_args()

  if config.verbose >= 2:
    logging.basicConfig(level = logging.DEBUG)
    os.environ['WINEDEBUG'] = ''
    xmake_verbose = ['-vD']
  elif config.verbose >= 1:
    logging.basicConfig(level = logging.INFO)
    os.environ['WINEDEBUG'] = 'fixme-all'
    xmake_verbose = ['-v']
  else:
    logging.basicConfig(level = logging.ERROR)
    os.environ['WINEDEBUG'] = '-all'
    xmake_verbose = []

  logging.info("testing GCC %s", config.branch)

  ver = resolve_profile(config)
  paths = ProjectPaths(config, ver)

  clean(config, paths)

  prepare_dirs(paths)

  prepare_test_binary(ver, paths)

  test_report = {
    'fail': False,
  }

  try:
    test_mingw_compiler(ver, paths, xmake_verbose)
    test_report['mingw64-compiler'] = "okay"
  except Exception as e:
    test_report['fail'] = True
    test_report['mingw64-compiler'] = repr(e)
  try:
    test_mingw_make_gdb(ver, paths)
    test_report['mingw64-make-gdb'] = "okay"
  except Exception as e:
    test_report['fail'] = True
    test_report['mingw64-make-gdb'] = repr(e)
  if config.enable_shared:
    try:
      test_mingw_shared(ver, paths, xmake_verbose)
      test_report['mingw64-shared'] = "okay"
    except Exception as e:
      test_report['fail'] = True
      test_report['mingw64-shared'] = repr(e)

  print("============================== TEST REPORT ==============================")
  pprint(test_report)

  if test_report['fail']:
    sys.exit(1)

if __name__ == '__main__':
  main()