#!/usr/bin/env python3 # Copyright Catch2 Authors # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE.txt or copy at # https://www.boost.org/LICENSE_1_0.txt) # SPDX-License-Identifier: BSL-1.0 import os import subprocess import sys import re import json cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)') def get_cmake_version(): result = subprocess.run(['cmake', '--version'], capture_output = True, check = True, text = True) version_match = cmake_version_regex.match(result.stdout) if not version_match: print('Could not find cmake version in output') print(f"output: '{result.stdout}'") exit(4) return (int(version_match.group(1)), int(version_match.group(2)), int(version_match.group(3))) def build_project(sources_dir, output_base_path, catch2_path): build_dir = os.path.join(output_base_path, 'ctest-registration-test') config_cmd = ['cmake', '-B', build_dir, '-S', sources_dir, f'-DCATCH2_PATH={catch2_path}', '-DCMAKE_BUILD_TYPE=Debug'] build_cmd = ['cmake', '--build', build_dir, '--config', 'Debug'] try: subprocess.run(config_cmd, capture_output = True, check = True, text = True) subprocess.run(build_cmd, capture_output = True, check = True, text = True) except subprocess.CalledProcessError as err: print('Error when building the test project') print(f'cmd: {err.cmd}') print(f'stderr: {err.stderr}') print(f'stdout: {err.stdout}') exit(3) return build_dir def get_test_names(build_path): # For now we assume that Windows builds are done using MSBuild under # Debug configuration. This means that we need to add "Debug" folder # to the path when constructing it. On Linux, we don't add anything. config_path = "Debug" if os.name == 'nt' else "" full_path = os.path.join(build_path, config_path, 'tests') cmd = [full_path, '--reporter', 'xml', '--list-tests'] result = subprocess.run(cmd, capture_output = True, check = True, text = True) import xml.etree.ElementTree as ET root = ET.fromstring(result.stdout) return [tc.text for tc in root.findall('TestCase/Name')] def get_ctest_listing(build_path): old_path = os.getcwd() os.chdir(build_path) cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] result = subprocess.run(cmd, capture_output = True, check = True, text = True) os.chdir(old_path) return result.stdout def extract_tests_from_ctest(ctest_output): ctest_response = json.loads(ctest_output) tests = ctest_response['tests'] test_names = [] for test in tests: test_command = test['command'] # First part of the command is the binary, second is the filter. # If there are less, registration has failed. If there are more, # registration has changed and the script needs updating. assert len(test_command) == 2 test_names.append(test_command[1]) test_name = test_command[1] return test_names def check_DL_PATHS(ctest_output): ctest_response = json.loads(ctest_output) tests = ctest_response['tests'] for test in tests: properties = test['properties'] for property in properties: if property['name'] == 'ENVIRONMENT_MODIFICATION': assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}" def escape_catch2_test_name(name): for char in ('\\', ',', '[', ']'): name = name.replace(char, f"\\{char}") return name if __name__ == '__main__': if len(sys.argv) != 3: print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') exit(2) catch2_path = sys.argv[1] output_base_path = sys.argv[2] sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) build_path = build_project(sources_dir, output_base_path, catch2_path) catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)] ctest_output = get_ctest_listing(build_path) ctest_test_names = extract_tests_from_ctest(ctest_output) mismatched = 0 for catch_test in catch_test_names: if catch_test not in ctest_test_names: print(f"Catch2 test '{catch_test}' not found in CTest") mismatched += 1 for ctest_test in ctest_test_names: if ctest_test not in catch_test_names: print(f"CTest test '{ctest_test}' not found in Catch2") mismatched += 1 if mismatched: print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!") exit(1) cmake_version = get_cmake_version() if cmake_version >= (3, 27): check_DL_PATHS(ctest_output)