From 94f1dc0f8d4fdf3be487a037975c35ff7572fa4d Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 18 Jan 2023 01:57:12 +0200 Subject: [PATCH 01/25] modules: The 'feature' module This module aims to provide a more dynamic way to control common features between compilers that would generally require special headers, arguments, or compile-time test cases, quite ideal to manage CPU features and solve compatibility issues. --- mesonbuild/modules/feature/__init__.py | 41 ++ mesonbuild/modules/feature/feature.py | 604 +++++++++++++++++++++++++ mesonbuild/modules/feature/module.py | 177 ++++++++ mesonbuild/modules/feature/utils.py | 67 +++ 4 files changed, 889 insertions(+) create mode 100644 mesonbuild/modules/feature/__init__.py create mode 100644 mesonbuild/modules/feature/feature.py create mode 100644 mesonbuild/modules/feature/module.py create mode 100644 mesonbuild/modules/feature/utils.py diff --git a/mesonbuild/modules/feature/__init__.py b/mesonbuild/modules/feature/__init__.py new file mode 100644 index 000000000000..6433d98eb442 --- /dev/null +++ b/mesonbuild/modules/feature/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import typing as T + +from .module import Module + +if T.TYPE_CHECKING: + from ...interpreter import Interpreter + +def initialize(interpreter: 'Interpreter') -> Module: + return Module() + diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/feature/feature.py new file mode 100644 index 000000000000..3767445dd449 --- /dev/null +++ b/mesonbuild/modules/feature/feature.py @@ -0,0 +1,604 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import typing as T +import re +from dataclasses import dataclass, field +from enum import IntFlag, auto + +from ... import mlog +from ...mesonlib import File, MesonException +from ...interpreter.type_checking import NoneType +from ...interpreterbase.decorators import ( + noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, + ContainerTypeInfo, +) +from .. import ModuleObject +from .utils import test_code, get_compiler + +if T.TYPE_CHECKING: + from typing import TypedDict + from typing_extensions import NotRequired + from ...interpreterbase import TYPE_var, TYPE_kwargs + from ...compilers import Compiler + from .. import ModuleState + +class FeatureSupport(IntFlag): + NONE: int = 0 + ARG: int = auto() + FILE: int = auto() + + def __str__(self) -> str: + return ', '.join(self.to_list()) + + def to_list(self) -> T.List[str]: + return [ + attr for attr in ('ARG', 'FILE') + if getattr(FeatureSupport, attr) in self + ] + +@dataclass(unsafe_hash=True, order=True) +class ImpliedAttr: + val: str = field(hash=True, compare=True) + match: T.Union[re.Pattern, None] = field( + default=None, hash=False, compare=False + ) + mfilter: T.Union[re.Pattern, None] = field( + default=None, hash=False, compare=False + ) + mjoin: str = field(default='', hash=False, compare=False) + + def __str__(self) -> str: + return self.val + + def copy(self) -> 'ImpliedAttr': + return ImpliedAttr(**self.__dict__) + + def to_dict(self) -> T.Dict[str, str]: + ret: T.Dict[str, str] = {} + for attr in ('val', 'mjoin'): + ret[attr] = getattr(self, attr) + for attr in ('match', 'mfilter'): + val = getattr(self, attr) + if not val: + val = '' + else: + val = str(val) + ret[attr] = val + return ret + + @staticmethod + def normalize(lst: 'T.List[ImpliedAttr]' + ) -> 'T.List[ImpliedAttr]': + ret: T.List[ImpliedAttr] = [] + for attr in lst: + if not attr.match: + if attr not in ret: + ret.append(attr) + continue + new_ret: T.List[ImpliedAttr] = [] + attr = attr.copy() + for ret_attr in ret: + if not attr.match.match(ret_attr.val): + new_ret.append(ret_attr) + continue + if not attr.mfilter: + continue + val = attr.mfilter.findall(ret_attr.val) + if not val: + continue + attr.val += ret_attr.mjoin.join(val) + if attr not in ret: + new_ret.append(attr) + ret = new_ret + return ret + +@dataclass +class TestResult: + #implicit: T.Set['FeatureObject'] = field(default_factory=set) + implicit: T.Set[T.Any] = field(default_factory=set) + args: T.List[ImpliedAttr] = field(default_factory=list) + detect: T.List[ImpliedAttr] = field(default_factory=list) + headers: T.List[ImpliedAttr] = field(default_factory=list) + defines: T.List[str] = field(default_factory=list) + undefines: T.List[str] = field(default_factory=list) + support: FeatureSupport = field( + default=FeatureSupport.ARG | FeatureSupport.FILE + ) + + def __add__(self, robj: 'TestResult') -> 'TestResult': + return TestResult( + implicit = self.implicit.union(robj.implicit), + args = ImpliedAttr.normalize(self.args + robj.args), + headers = ImpliedAttr.normalize(self.headers + robj.headers), + detect = ImpliedAttr.normalize(self.detect + robj.detect), + defines = self.defines + [ + v for v in robj.defines + if v not in self.defines + ], + undefines = self.undefines + [ + v for v in robj.undefines + if v not in self.undefines + ], + support = self.support & robj.support + ) + + def to_dict(self) -> T.Dict[str, T.List[str]]: + ret = {} + for attr in ('args', 'detect', 'headers'): + ret[attr] = [str(v) for v in getattr(self, attr)] + + for attr in ('defines', 'undefines'): + ret[attr] = getattr(self, attr)[:] + + ret['implicit'] = [str(fet) for fet in sorted(self.implicit)] + ret['support'] = self.support.to_list() + return ret + + +if T.TYPE_CHECKING: + IMPLIED_ATTR = T.Union[ + None, str, T.Dict[str, str], T.List[ + T.Union[str, T.Dict[str, str]] + ] + ] + +class Convert: + @staticmethod + def _implattr(opt_name: str, values: 'IMPLIED_ATTR', + ) -> T.Union[None, T.List[ImpliedAttr]]: + if values is None: + return None + ret: T.List[ImpliedAttr] = [] + values = [values] if isinstance(values, (str, dict)) else values + accepted_keys = ('val', 'match', 'mfilter', 'mjoin') + for edict in values: + if isinstance(edict, str): + ret.append(ImpliedAttr(val=edict)) + continue + if not isinstance(edict, dict): + # It shouldn't happen + continue + unknown_keys = tuple([k for k in edict.keys() if k not in accepted_keys]) + if unknown_keys: + raise MesonException( + f'feature.new: unknown keys {unknown_keys} in ' + f'option {opt_name}' + ) + val = edict.get('val') + if val is None: + raise MesonException( + f'feature.new: option "{opt_name}" requires ' + f'a dictionary with key "val" to be set' + ) + implattr = ImpliedAttr(val=val, mjoin=edict.get('mjoin', '')) + for cattr in ('match', 'mfilter'): + cval = edict.get(cattr) + if not cval: + continue + try: + ccval = re.compile(cval) + except Exception as e: + raise MesonException( + 'feature.new: unable to ' + f'compile the regex in option "{opt_name}"\n' + f'"{cattr}:{cval}" -> {str(e)}' + ) + setattr(implattr, cattr, ccval) + ret.append(implattr) + return ret + + @staticmethod + def implattr(opt_name: str) -> T.Callable[ + ['IMPLIED_ATTR'], + T.Union[None, T.List[ImpliedAttr]]]: + return lambda values: Convert._implattr(opt_name, values) + +if T.TYPE_CHECKING: + class FeatureKwArgs(TypedDict): + #implies: T.Optional[T.List['FeatureObject']] + implies: NotRequired[T.List[T.Any]] + group: NotRequired[T.List[str]] + detect: NotRequired[T.List[ImpliedAttr]] + args: NotRequired[T.List[ImpliedAttr]] + headers: NotRequired[T.List[ImpliedAttr]] + test_code: NotRequired[T.Union[str, File]] + extra_tests: NotRequired[T.Dict[str, T.Union[str, File]]] + disable: NotRequired[str] + + class FeatureUpdateKwArgs(FeatureKwArgs): + name: NotRequired[str] + interest: NotRequired[int] + +class FeatureObject(ModuleObject): + name: str + interest: int + implies: T.Set['FeatureObject'] + group: T.List[str] + detect: T.List[ImpliedAttr] + args: T.List[ImpliedAttr] + headers: T.List[ImpliedAttr] + test_code: T.Union[str, File] + extra_tests: T.Dict[str, T.Union[str, File]] + disable: str + + def __init__(self, state: 'ModuleState', + args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs') -> None: + + IMPLIED_ATTR_TYPES = ( + str, ContainerTypeInfo(dict, str), + ContainerTypeInfo(list, (dict, str)), + ) + @typed_pos_args('feature.new', str, int) + @typed_kwargs('feature.new', + KwargInfo('implies', + (FeatureObject, ContainerTypeInfo(list, FeatureObject)), + default=[], listify=True + ), + KwargInfo('group', + (str, ContainerTypeInfo(list, str)), + default=[], listify=True + ), + KwargInfo('detect', + IMPLIED_ATTR_TYPES, convertor=Convert.implattr('detect'), + default=[] + ), + KwargInfo('args', + IMPLIED_ATTR_TYPES, convertor=Convert.implattr('args'), + default=[] + ), + KwargInfo('headers', + IMPLIED_ATTR_TYPES, convertor=Convert.implattr('headers'), + default=[] + ), + KwargInfo( + 'test_code', (str, File), + default='' + ), + KwargInfo('extra_tests', + (ContainerTypeInfo(dict, (str, File))), + default={} + ), + KwargInfo('disable', (str), default=''), + ) + def init_attrs(state: 'ModuleState', + args: T.Tuple[str, int], + kwargs: 'FeatureKwArgs' + ) -> None: + self.name = args[0] + self.interest = args[1] + self.implies = set(kwargs['implies']) + self.group = kwargs['group'] + self.detect = kwargs['detect'] + self.args = kwargs['args'] + self.headers = kwargs['headers'] + self.test_code = kwargs['test_code'] + self.extra_tests = kwargs['extra_tests'] + self.disable: str = kwargs['disable'] + + super().__init__() + init_attrs(state, args, kwargs) + self.methods.update({ + 'update': self.update_method, + 'to_dict': self.to_dict_method, + }) + + def __hash__(self) -> int: + return hash(str(id(self)) + self.name) + + def __eq__(self, robj: object) -> bool: + if not isinstance(robj, FeatureObject): + return False + return self is robj and self.name == robj.name + + def __lt__(self, robj: object) -> T.Any: + if not isinstance(robj, FeatureObject): + return NotImplemented + return self.interest < robj.interest + + def __le__(self, robj: object) -> T.Any: + if not isinstance(robj, FeatureObject): + return NotImplemented + return self.interest <= robj.interest + + def __gt__(self, robj: object) -> T.Any: + return robj < self + + def __ge__(self, robj: object) -> T.Any: + return robj <= self + + def __str__(self) -> str: + return self.name + + def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs') -> 'FeatureObject': + IMPLIED_ATTR_NTYPES = ( + NoneType, str, ContainerTypeInfo(dict, str), + ContainerTypeInfo(list, (dict, str)), + ) + @noPosargs + @typed_kwargs('feature.update', + KwargInfo('name', (NoneType, str)), + KwargInfo('interest', (NoneType, int)), + KwargInfo('implies', + (NoneType, FeatureObject, ContainerTypeInfo(list, FeatureObject)), + listify=True + ), + KwargInfo('group', + (NoneType, str, ContainerTypeInfo(list, str)), listify=True + ), + KwargInfo('detect', + IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('detect') + ), + KwargInfo('args', + IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('args') + ), + KwargInfo('headers', + IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('headers') + ), + KwargInfo('test_code', (NoneType, str, File)), + KwargInfo('extra_tests', + (NoneType, ContainerTypeInfo(dict, (str, File))) + ), + KwargInfo('disable', (NoneType, str)), + ) + def update(state: 'ModuleState', args: T.List['TYPE_var'], + kwargs: 'FeatureUpdateKwArgs') -> None: + for k, v in kwargs.items(): + if v is not None and k != 'implies': + setattr(self, k, v) + implies = kwargs.get('implies') + if implies is not None: + self.implies = set(implies) + update(state, args, kwargs) + return self + + @noPosargs + @noKwargs + def to_dict_method(self, state: 'ModuleState', args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs' + ) -> T.Dict[str, T.Union[str, T.List[str]]]: + return self.to_dict() + + def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str]]]: + ret = self.__dict__.copy() + ret.pop('methods') + ret['implies'] = [str(fet) for fet in sorted(self.implies)] + for attr in ('detect', 'args', 'headers'): + ret[attr] = [v.to_dict() for v in getattr(self, attr)] + return ret + + def get_implicit(self, _caller: 'T.Set[FeatureObject]' = None + ) -> 'T.Set[FeatureObject]': + # infinity recursive guard since + # features can imply each other + _caller = {self,} if not _caller else _caller.union({self,}) + implies = self.implies.difference(_caller) + ret = self.implies + for sub_fet in implies: + ret = ret.union(sub_fet.get_implicit(_caller)) + return ret + + def test_args(self, state: 'ModuleState', compiler: 'Compiler' + ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: + check_args = compiler.has_multi_arguments + args: T.List[ImpliedAttr] = [] + skipped_args: T.List[ImpliedAttr] = [] + cached = True + for a in self.args: + result, tcached = check_args([a.val], state.environment) + if result: + args.append(a) + else: + skipped_args.append(a) + cached &= tcached + return cached, args, skipped_args + + def test_headers(self, state: 'ModuleState', compiler: 'Compiler', + args: T.List[str] + ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: + check_header = compiler.check_header + headers: T.List[ImpliedAttr] = [] + skipped_headers: T.List[ImpliedAttr] = [] + cached = True + for h in self.headers: + result, tcached = check_header( + hname=h.val, prefix='', env=state.environment, + extra_args=args + ) + if result: + headers.append(h) + else: + skipped_headers.append(h) + cached &= tcached + return cached, headers, skipped_headers + + def test_extra(self, state: 'ModuleState', + compiler: 'Compiler', + args: T.List[str], + headers: T.List[str] + ) -> T.Tuple[bool, T.List[str], T.List[str]]: + extra: T.List[str] = [] + skipped_extra: T.List[str] = [] + cached = True + for extra_name, extra_test in self.extra_tests.items(): + tcached, test, stderr = test_code( + state, compiler, args, headers, extra_test + ) + if test: + extra.append(extra_name) + else: + skipped_extra.append(extra_name) + cached &= tcached + return cached, extra, skipped_extra + + def test(self, state: 'ModuleState', compiler: 'Compiler', + force_args: T.Optional[T.List[str]] = None + ) -> TestResult: + + cached, disabled, error, result = self.test_impl( + state, compiler, force_args + ) + log_prefix = f'Test feature "{mlog.bold(self.name)}" :' + cached_msg = f'({mlog.blue("cached")})'if cached else '' + if not result: + reason = ( + mlog.yellow('Disabled') if disabled + else mlog.red('Unsupported') + ) + mlog.log( + log_prefix, + reason, + cached_msg + ) + mlog.debug( + log_prefix, + reason, + 'due to', + error + ) + return TestResult(support=FeatureSupport.NONE) + mlog.log( + log_prefix, + mlog.green('Supported'), + cached_msg + ) + return result + + def test_impl(self, state: 'ModuleState', compiler: 'Compiler', + force_args: T.Optional[T.List[str]] = None, + _caller: T.Optional[T.Set['FeatureObject']] = None, + ) -> T.Tuple[ + bool, bool, str, T.Optional[TestResult] + ]: + + prefix = f'implied feature "{self.name}" ' if _caller else '' + if self.disable: + return False, True, f'{prefix}disabled due to {self.disable}', None + + _caller = {self,} if not _caller else _caller.union({self,}) + cached = True + result = TestResult(implicit=self.implies.difference(_caller)) + after = TestResult() + implicit: T.List[FeatureObject] = sorted(result.implicit) + for fet in implicit: + imp_ret = fet.test_impl(state, compiler, force_args, _caller) + imp_cached, _, _, imp_result = imp_ret + if not imp_result: + return imp_ret + if fet > self: + after += imp_result + else: + result += imp_result + cached &= imp_cached + + if force_args is None: + tcached, rargs, skipped_args = self.test_args(state, compiler) + if (not rargs and skipped_args) and not self.test_code: + return ( + tcached, False, + f'{prefix}specified arguments {tuple(skipped_args)} are not ' + 'supported and the test file is not specified', + None + ) + result.args = ImpliedAttr.normalize( + result.args + rargs + after.args + ) + cached &= tcached + args = [a.val for a in result.args] + else: + args = force_args + + tcached, rheaders, skipped_headers = self.test_headers(state, compiler, args) + if not rheaders and skipped_headers: + return ( + tcached, False, + f'{prefix}specified headers {tuple(h.val for h in skipped_headers)} ' + 'are not supported by the compiler', + None + ) + cached &= tcached + result.headers = ImpliedAttr.normalize( + result.headers + rheaders + after.headers + ) + headers = [h.val for h in result.headers] + + if self.test_code: + tcached, test, stderr = test_code( + state, compiler, args, headers, self.test_code + ) + if not test: + return ( + tcached, False, + f'{prefix}compiler was not able to compile the test code\n' + f'Arguments: {str(args)}\n' + f'Headers: {str(headers)}\n' + f':\n"{stderr}"', + None + ) + cached &= tcached + + if not self.detect: + detect = [ + ImpliedAttr(val=name) for name in ( + self.group if self.group else [self.name] + ) + ] + else: + detect = self.detect + result.detect = ImpliedAttr.normalize( + result.detect + detect + after.detect + ) + + tcached, extra, skipped_extra = self.test_extra( + state, compiler, args, headers + ) + cached &= tcached + + result.defines += [ + df for df in [self.name] + self.group + extra + after.defines + if df not in result.defines + ] + result.undefines += [ + udf for udf in skipped_extra + after.undefines + if udf not in result.undefines + ] + + support = FeatureSupport.NONE + if args and not skipped_args: + support = FeatureSupport.ARG + if self.test_code: + support |= FeatureSupport.FILE + support &= after.support + + result.support &= support + return cached, False, '', result diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py new file mode 100644 index 000000000000..503ecf38c3e3 --- /dev/null +++ b/mesonbuild/modules/feature/module.py @@ -0,0 +1,177 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import typing as T + +from ...compilers import Compiler +from ...interpreter.type_checking import NoneType +from ...interpreterbase.decorators import ( + noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, + ContainerTypeInfo, +) +from .. import ModuleInfo, NewExtensionModule, ModuleReturnValue + +from .feature import FeatureObject +from .utils import get_compiler + +if T.TYPE_CHECKING: + from typing import TypedDict + from ...interpreterbase import TYPE_var, TYPE_kwargs + from .. import ModuleState + from .feature import FeatureKwArgs + + class TestKwArgs(TypedDict): + compiler: T.Optional[Compiler] + force_args: T.Optional[T.List[str]] + any: T.Optional[bool] + +class Module(NewExtensionModule): + INFO = ModuleInfo('feature', '0.1.0') + + def __init__(self) -> None: + super().__init__() + self.methods.update({ + 'new': self.new_method, + 'cpu_features': self.cpu_features_method, + 'test': self.test_method, + 'implicit': self.implicit_method, + 'ahead': self.ahead_method, + 'untied': self.untied_method, + }) + + def new_method(self, state: 'ModuleState', + args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs') -> FeatureObject: + return FeatureObject(state, args, kwargs) + + @typed_pos_args('feature.test', + varargs=FeatureObject + ) + @typed_kwargs('feature.test', + KwargInfo('compiler', + (NoneType, Compiler) + ), + KwargInfo('force_args', + (NoneType, str, ContainerTypeInfo(list, str)), + listify=True + ), + KwargInfo('any', + bool, default=True + ) + ) + def test_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TestKwArgs' + ) -> T.List[T.Union[bool, T.Dict[str, T.Any]]]: + + compiler = kwargs.get('compiler') + force_args = kwargs.get('force_args') + any_ = kwargs['any'] + + if not compiler: + compiler = get_compiler(state) + + features = args[0] + if len(features) == 1: + result = features[0].test(state, compiler, force_args) + if result is None: + return [False, {}] + return [True, result.to_dict()] + + features = sorted(features) + result = features[0].test(state, compiler, force_args) + for fet in features[1:]: + ret = fet.test(state, compiler, force_args) + if not ret: + if not any_: + return None + continue + result += ret + if result is None: + return [False, {}] + return [True, result.to_dict()] + + @typed_pos_args('feature.implicit', + varargs=FeatureObject + ) + @noKwargs + def implicit_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + + features = args[0] + implicit = set().union(*[fet.get_implicit() for fet in features]) + # since features can imply each other + implicit.difference_update(set(features)) + return sorted(implicit) + + @typed_pos_args('feature.ahead', + varargs=FeatureObject + ) + @noKwargs + def ahead_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + + features = args[0] + implicit = set().union(*[fet.get_implicit() for fet in features]) + ahead = [fet for fet in features if fet not in implicit] + if len(ahead) == 0: + # return the highest interested feature + # if all features imply each other + ahead = list(sorted(features, reverse=True)[:1]) + return ahead + + @typed_pos_args('feature.untied', + varargs=FeatureObject + ) + @noKwargs + def untied_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + features = args[0] + ret: T.List[FeatureObject] = [] + for fet in features: + tied = { + sub_fet for sub_fet in ret + if sub_fet in fet.implies and fet in sub_fet.implies + } + if tied: + stied = sorted(tied.union({fet,})) + if fet not in stied[1:]: + continue + ret.remove(stied[0]) + ret.append(fet) + return ret + diff --git a/mesonbuild/modules/feature/utils.py b/mesonbuild/modules/feature/utils.py new file mode 100644 index 000000000000..b51ad5fbbfd2 --- /dev/null +++ b/mesonbuild/modules/feature/utils.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import typing as T + +from ...mesonlib import MesonException, MachineChoice + +if T.TYPE_CHECKING: + from ...compilers import Compiler + from ...mesonlib import File + from .. import ModuleState + +def get_compiler(state: 'ModuleState') -> 'Compiler': + for_machine = MachineChoice.BUILD + clist = state.environment.coredata.compilers[for_machine] + for cstr in ('c', 'cpp'): + try: + compiler = clist[cstr] + break + except KeyError: + raise MesonException( + 'Unable to get compiler for C or C++ language ' + 'try to specify a valid C/C++ compiler via option "compiler".' + ) + return compiler + +def test_code(state: 'ModuleState', compiler: 'Compiler', + args: T.List[str], headers: T.List[str], + code: 'T.Union[str, File]' + ) -> T.Tuple[bool, bool, str]: + if isinstance(code, str): + heads = '\n'.join([f'#include <{h}>' for h in headers]) + code = heads + '\n' + code + # TODO: treat warnings as errors + with compiler.cached_compile( + code, state.environment.coredata, extra_args=args + ) as p: + return p.cached, p.returncode == 0, p.stderr + From 86eb63992fa5c9ecba7fe4ee0c3fde8a9faed867 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 18 Jan 2023 01:58:21 +0200 Subject: [PATCH 02/25] modules: Add method 'feature.cpu_features' and brings X86 CPU features --- mesonbuild/modules/feature/module.py | 22 + mesonbuild/modules/feature/x86_features.py | 552 +++++++++++++++++++++ 2 files changed, 574 insertions(+) create mode 100644 mesonbuild/modules/feature/x86_features.py diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py index 503ecf38c3e3..243f74ffe388 100644 --- a/mesonbuild/modules/feature/module.py +++ b/mesonbuild/modules/feature/module.py @@ -41,6 +41,7 @@ from .feature import FeatureObject from .utils import get_compiler +from .x86_features import x86_features if T.TYPE_CHECKING: from typing import TypedDict @@ -72,6 +73,27 @@ def new_method(self, state: 'ModuleState', kwargs: 'TYPE_kwargs') -> FeatureObject: return FeatureObject(state, args, kwargs) + @noPosargs + @typed_kwargs('feature.cpu_features', + KwargInfo( + 'compiler', (NoneType, Compiler), + ), + ) + def cpu_features_method(self, state: 'ModuleState', + args: T.List['TYPE_var'], + kwargs: T.Dict[str, Compiler] + ) -> T.Dict[str, FeatureObject]: + compiler = kwargs['compiler'] + if compiler is None: + compiler = get_compiler(state) + + features: T.Dict[str, FeatureObject] = {} + for func in ( + x86_features, + ): + features.update(func(state, compiler)) + return features + @typed_pos_args('feature.test', varargs=FeatureObject ) diff --git a/mesonbuild/modules/feature/x86_features.py b/mesonbuild/modules/feature/x86_features.py new file mode 100644 index 000000000000..c5eadad1bf85 --- /dev/null +++ b/mesonbuild/modules/feature/x86_features.py @@ -0,0 +1,552 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * Neither the name of the NumPy Developers nor the names of any +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import typing as T +from textwrap import dedent + +from ... import mlog +from .feature import FeatureObject + +if T.TYPE_CHECKING: + from ...compilers import Compiler + from ...interpreterbase import TYPE_var, TYPE_kwargs + from .. import ModuleState + +def test_code_main(code: str) -> str: + return dedent(f'''\ + int main(int, char **argv) + {{ + char *src = argv[1]; + {dedent(code)} + return 0; + }} + ''') + +def test_code(NAME: str, code: str) -> str: + return dedent(f'''\ + #if defined(DETECT_FEATURES) && defined(__{NAME}__) + #error "HOST/ARCH doesn't support {NAME}" + #endif + {test_code_main(code)} + ''') + +def feature(state: 'ModuleState', name: str, interest: int, + kwargs: 'TYPE_kwargs') -> FeatureObject: + code = kwargs.get('test_code') + extra_tests = kwargs.get('extra_tests') + if isinstance(code, str): + kwargs['test_code'] = test_code(name, code) + if isinstance(extra_tests, dict): + kwargs['extra_tests'] = { + tname: test_code_main(code) if isinstance(code, str) else code + for tname, code in extra_tests.items() + } + return FeatureObject(state, [name, interest], kwargs) + +def _init_features(state: 'ModuleState') -> T.Dict[str, FeatureObject]: + features: T.Dict[str, FeatureObject] = {} + fet: T.Callable[[str, int, 'TYPE_kwargs'], FeatureObject] = \ + lambda name, interest, kwargs: features.setdefault( + name, feature(state, name, interest, kwargs) + ) + + cpu_family = state.build_machine.cpu_family + is_x86 = cpu_family in ('x86', 'x86_64', 'x64') + SSE = fet('SSE', 1, dict( + headers='xmmintrin.h', + disable=( + f'not supported by build machine "{cpu_family}"' + if not is_x86 else '' + ), + test_code='''\ + __m128 s0 = _mm_loadu_ps((float*)src); + __m128 s1 = _mm_loadu_ps((float*)src+4); + __m128 rt = _mm_add_ps(s0, s1); + _mm_storeu_ps((float*)src, rt); + ''' + )) + SSE2 = fet('SSE2', 2, dict( + implies=SSE, headers='emmintrin.h', + test_code='''\ + __m128i s0 = _mm_loadu_si128((__m128i*)src); + __m128i s1 = _mm_loadu_si128((__m128i*)src+16); + __m128i rt = _mm_add_epi16(s0, s1); + _mm_storeu_si128((__m128i*)src, rt); + ''' + )) + SSE3 = fet('SSE3', 3, dict( + implies=SSE2, headers='pmmintrin.h', + test_code='''\ + __m128 s0 = _mm_loadu_ps((float*)src); + __m128 s1 = _mm_loadu_ps((float*)src+4); + __m128 rt = _mm_hadd_ps(s0, s1); + _mm_storeu_ps((float*)src, rt); + ''' + )) + if state.build_machine.is_64_bit: + SSE.update_method(state, [], dict(implies=[SSE2, SSE3])) + SSE2.update_method(state, [], dict(implies=[SSE, SSE3])) + + SSSE3 = fet('SSSE3', 4, dict( + implies=SSE3, headers='tmmintrin.h', + test_code='''\ + __m128i s0 = _mm_loadu_si128((__m128i*)src); + __m128i s1 = _mm_loadu_si128((__m128i*)src+16); + __m128i rt = _mm_hadd_epi16(s0, s1); + _mm_storeu_si128((__m128i*)src, rt); + ''' + )) + SSE41 = fet('SSE41', 5, dict( + implies=SSSE3, headers='smmintrin.h', + test_code='''\ + __m128 s0 = _mm_loadu_ps((float*)src); + __m128 rt = _mm_ceil_ps(s0); + _mm_storeu_ps((float*)src, rt); + ''' + )) + POPCNT = fet('POPCNT', 6, dict( + implies=SSE41, headers='popcntintrin.h', + test_code='''\ + unsigned long long a = *((unsigned long long*)src); + unsigned int b = *((unsigned int*)src+4); + int rt; + #if defined(_M_X64) || defined(__x86_64__) + a = _mm_popcnt_u64(a); + #endif + b = _mm_popcnt_u32(b); + rt = (int)a + (int)b; + _mm_storeu_si128((__m128i*)src, _mm_set1_epi32(rt)); + ''' + )) + SSE42 = fet('SSE42', 7, dict( + implies=POPCNT, + test_code='''\ + __m128i s0 = _mm_loadu_si128((__m128i*)src); + __m128i s1 = _mm_loadu_si128((__m128i*)src+16); + __m128i rt = _mm_cmpgt_epi64(s0, s1); + _mm_storeu_si128((__m128i*)src, rt); + ''' + )) + # 7-20 left as margin for any extra features + AVX = fet('AVX', 20, dict( + implies=SSE42, + headers='immintrin.h', + detect=dict(val='AVX', match='.*'), + test_code='''\ + __m256 s0 = _mm256_loadu_ps((float*)src); + __m256 s1 = _mm256_loadu_ps((float*)src+8); + __m256 rt = _mm256_add_ps(s0, s1); + _mm256_storeu_ps((float*)src, rt); + ''' + )) + # Amd abandon these two features, should we remove them? + XOP = fet('XOP', 21, dict( + implies=AVX, headers='x86intrin.h', + test_code='''\ + __m128i s0 = _mm_loadu_si128((__m128i*)src); + __m128i s1 = _mm_loadu_si128((__m128i*)src+16); + __m128i rt = _mm_comge_epu32(s0, s1); + _mm_storeu_si128((__m128i*)src, rt); + ''' + )) + FMA4 = fet('FMA4', 22, dict( + implies=AVX, headers='x86intrin.h', + test_code='''\ + __m128 s0 = _mm_loadu_ps((float*)src); + __m128 s1 = _mm_loadu_ps((float*)src+4); + __m128 s2 = _mm_loadu_ps((float*)src+8); + __m128 rt = _mm256_macc_ps(s0, s1, s2); + _mm_storeu_ps((float*)src, rt); + ''' + )) + # x86 half-precision + F16C = fet('F16C', 23, dict( + implies=AVX, + test_code=f'''\ + __m128i s0 = _mm_loadu_si128((__m128i*)src); + __m128 rt = _mm_cvtph_ps(s0); + _mm_storeu_ps((float*)src, rt); + ''' + )) + FMA3 = fet('FMA3', 24, dict( + implies=F16C, + test_code=f'''\ + __m256 s0 = _mm256_loadu_ps((float*)src); + __m256 s1 = _mm256_loadu_ps((float*)src + 8); + __m256 s2 = _mm256_loadu_ps((float*)src + 16); + __m256 rt = _mm256_fmadd_ps(s0, s1, s2); + _mm256_storeu_ps((float*)src, rt); + ''' + )) + AVX2 = fet('AVX2', 25, dict( + implies=F16C, + test_code=f'''\ + __m256i s0 = _mm256_loadu_si256((__m256i*)src); + __m256i rt = _mm256_abs_epi16(s0); + _mm256_storeu_si256((__m256i*)src, rt); + ''' + )) + # 25-40 left as margin for any extra features + AVX512_COMMON = fet('AVX512_COMMON', 40, dict( + implies=[FMA3, AVX2], + group = ['AVX512F', 'AVX512CD'], + detect = [ + dict(val='AVX512F', match='.*'), + 'AVX512CD' + ], + test_code=f'''\ + __m512i s0 = _mm512_loadu_si512((__m512i*)src); + __m512i rt = _mm512_abs_epi32(s0); + /* avx512cd */ + rt = _mm512_lzcnt_epi32(rt); + _mm512_storeu_si512((__m512i*)src, rt); + ''', + extra_tests = dict( + AVX512F_REDUCE=f'''\ + __m512i si = _mm512_loadu_si512((__m512i*)src); + __m512 ps = _mm512_loadu_ps((__m512*)src); + __m512d pd = _mm512_loadu_pd((__m512d*)src); + /* add */ + float sum_ps = _mm512_reduce_add_ps(ps); + double sum_pd = _mm512_reduce_add_pd(pd); + int sum_int = (int)_mm512_reduce_add_epi64(si); + sum_int += (int)_mm512_reduce_add_epi32(si); + /* mul */ + sum_ps += _mm512_reduce_mul_ps(ps); + sum_pd += _mm512_reduce_mul_pd(pd); + sum_int += (int)_mm512_reduce_mul_epi64(si); + sum_int += (int)_mm512_reduce_mul_epi32(si); + /* min */ + sum_ps += _mm512_reduce_min_ps(ps); + sum_pd += _mm512_reduce_min_pd(pd); + sum_int += (int)_mm512_reduce_min_epi32(si); + sum_int += (int)_mm512_reduce_min_epu32(si); + sum_int += (int)_mm512_reduce_min_epi64(si); + /* max */ + sum_ps += _mm512_reduce_max_ps(ps); + sum_pd += _mm512_reduce_max_pd(pd); + sum_int += (int)_mm512_reduce_max_epi32(si); + sum_int += (int)_mm512_reduce_max_epu32(si); + sum_int += (int)_mm512_reduce_max_epi64(si); + /* and */ + sum_int += (int)_mm512_reduce_and_epi32(si); + sum_int += (int)_mm512_reduce_and_epi64(si); + /* or */ + sum_int += (int)_mm512_reduce_or_epi32(si); + sum_int += (int)_mm512_reduce_or_epi64(si); + sum_int += (int)sum_ps + (int)sum_pd; + *(int*)src = sum_int; + ''' + ) + )) + AVX512_KNL = fet('AVX512_KNL', 41, dict( + implies=AVX512_COMMON, + group = ['AVX512ER', 'AVX512PF'], + test_code=f'''\ + int base[128]; + __m512d ad = _mm512_loadu_pd((__m512d*)src); + /* ER */ + __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); + /* PF */ + _mm512_mask_prefetch_i64scatter_pd( + base, _mm512_cmpeq_epi64_mask(a, a), a, 1, _MM_HINT_T1 + ); + *(int*)src = base[0]; + ''' + )) + AVX512_KNM = fet('AVX512_KNM', 42, dict( + implies=AVX512_KNL, + group = ['AVX5124FMAPS', 'AVX5124VNNIW', 'AVX512VPOPCNTDQ'], + test_code=f'''\ + __m512i si = _mm512_loadu_si512((__m512i*)src); + __m512 ps = _mm512_loadu_ps((__m512*)src + 64); + /* 4FMAPS */ + ps = _mm512_4fmadd_ps(ps, ps, ps, ps, ps, NULL); + /* 4VNNIW */ + si = _mm512_4dpwssd_epi32(si, si, si, si, si, NULL); + /* VPOPCNTDQ */ + si = _mm512_popcnt_epi64(si); + si = _mm512_add_epi32(si, _mm512_castps_si512(ps)); + _mm512_storeu_si512((__m512i*)src, si); + ''' + )) + AVX512_SKX = fet('AVX512_SKX', 43, dict( + implies=AVX512_COMMON, + group = ['AVX512VL', 'AVX512BW', 'AVX512DQ'], + test_code=f'''\ + __m512i aa = _mm512_abs_epi32(_mm512_loadu_si512((__m512i*)src)); + /* VL */ + __m256i a = _mm256_abs_epi64(_mm512_extracti64x4_epi64(aa, 1)); + /* DQ */ + __m512i b = _mm512_broadcast_i32x8(a); + /* BW */ + b = _mm512_abs_epi16(b); + _mm512_storeu_si512((__m512i*)src, b); + ''', + extra_tests = dict( + AVX512BW_MASK=f'''\ + __m512i s0 = _mm512_loadu_si512((__m512i*)src); + __m512i s1 = _mm512_loadu_si512((__m512i*)src + 64); + __mmask64 m64 = _mm512_cmpeq_epi8_mask(s0, s1); + m64 = _kor_mask64(m64, m64); + m64 = _kxor_mask64(m64, m64); + m64 = _cvtu64_mask64(_cvtmask64_u64(m64)); + m64 = _mm512_kunpackd(m64, m64); + m64 = (__mmask64)_mm512_kunpackw((__mmask32)m64, (__mmask32)m64); + *(int*)src = (int)_cvtmask64_u64(m64); + ''', + AVX512DQ_MASK=f'''\ + __m512i s0 = _mm512_loadu_si512((__m512i*)src); + __m512i s1 = _mm512_loadu_si512((__m512i*)src + 64); + __mmask8 m8 = _mm512_cmpeq_epi64_mask(s0, s1); + m8 = _kor_mask8(m8, m8); + m8 = _kxor_mask8(m8, m8); + m8 = _cvtu32_mask8(_cvtmask8_u32(m8)); + *(int*)src = (int)_cvtmask8_u32(m8); + ''' + ) + )) + AVX512_CLX = fet('AVX512_CLX', 44, dict( + implies=AVX512_SKX, + group = 'AVX512VNNI', + test_code=f'''\ + /* VNNI */ + __m512i a = _mm512_loadu_si512((__m512i*)src); + a = _mm512_dpbusd_epi32(a, _mm512_setzero_si512(), a); + _mm512_storeu_si512((__m512i*)src, a); + ''' + )) + AVX512_CNL = fet('AVX512_CNL', 45, dict( + implies=AVX512_SKX, + group = ['AVX512IFMA', 'AVX512VBMI'], + test_code=f'''\ + __m512i a = _mm512_loadu_si512((const __m512i*)src); + /* IFMA */ + a = _mm512_madd52hi_epu64(a, a, _mm512_setzero_si512()); + /* VMBI */ + a = _mm512_permutex2var_epi8(a, _mm512_setzero_si512(), a); + _mm512_storeu_si512((__m512i*)src, a); + ''' + )) + AVX512_ICL = fet('AVX512_ICL', 46, dict( + implies=[AVX512_CLX, AVX512_CNL], + group = ['AVX512VBMI2', 'AVX512BITALG', 'AVX512VPOPCNTDQ'], + test_code=f'''\ + __m512i a = _mm512_loadu_si512((__m512i*)src); + /* VBMI2 */ + a = _mm512_shrdv_epi64(a, a, _mm512_setzero_si512()); + /* BITLAG */ + a = _mm512_popcnt_epi8(a); + /* VPOPCNTDQ */ + a = _mm512_popcnt_epi64(a); + _mm512_storeu_si512((__m512i*)src, a); + ''' + )) + return features + +def _features_args(state: 'ModuleState', + compiler: 'Compiler', + features: T.Dict[str, FeatureObject]) -> None: + cid = compiler.get_id() + # for both unix-like and msvc-like + if 'intel' in cid: + # Intel Compiler doesn't support AVX2 or FMA3 independently + for fet, implies in ( + 'FMA3', ('F16C', 'AVX2'), + 'AVX2', ('F16C', 'FMA3'), + ): + features[fet].update_method(state, [], { + 'implies': [features[sub_fet] for sub_fet in implies] + }) + for dis_fet in ('XOP', 'FMA4'): + features[dis_fet].update_method(state, [], { + 'disable': 'Intel Compiler does not support it' + }) + if 'intel-cl' in cid: + for fet in ( + 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'AVX' + ): + features[fet].update_method(state, [], { + 'args': dict(val='/arch:' + fet, match='/arch:.*') + }) + # POPCNT, and F16C don't own private FLAGS still + # Intel's compiler provides ABI capability for them. + for fet, arg in ( + ('SSE42', dict( + val='/arch:SSE4.2', match='/arch:.*' + )), + ('FMA3', dict( + val='/arch:CORE-AVX2', match='/arch:.*' + )), + ('AVX2', dict( + val='/arch:CORE-AVX2', match='/arch:.*' + )), + ('AVX512_COMMON', dict( + val='/Qx:COMMON-AVX512', match='/arch:.*' + )), + ('AVX512_KNL', dict( + val='/Qx:KNL', match='/[arch|Qx]:.*' + )), + ('AVX512_KNM', dict( + val='/Qx:KNM', match='/[arch|Qx]:.*' + )), + ('AVX512_SKX', dict( + val='/Qx:SKYLAKE-AVX512', match='/[arch|Qx]:.*' + )), + ('AVX512_CLX', dict( + val='/Qx:CASCADELAKE', match='/[arch|Qx]:.*' + )), + ('AVX512_CNL', dict( + val='/Qx:CANNONLAKE', match='/[arch|Qx]:.*' + )), + ('AVX512_ICL', dict( + val='/Qx:ICELAKE-CLIENT', match='/[arch|Qx]:.*' + )), + ): + features[fet].update_method(state, [], {'args': arg}) + elif 'intel' in cid: + for fet in ( + 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'AVX' + ): + features[fet].update_method(state, [], { + 'args': '-m' + fet.lower() + }) + # POPCNT, and F16C don't own private FLAGS still + # Intel's compiler provides ABI capability for them. + + # we specify the arguments that we need to filterd + # from implied features rather than blinedly removes all args(.*) + # since args may updated from outside. + filter_all = ".*m[sse|avx|arch\=|-x[a-z0-9\-]].*" + for fet, arg in ( + ('SSE41', dict(val='-msse4.1')), + ('SSE42', dict(val='-msse4.2')), + ('FMA3', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), + ('AVX2', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), + ('AVX512_COMMON', dict( + val='-march=common-avx512', match='.*m[sse|avx|arch\=].*' + )), + ('AVX512_KNL', dict( + val='-xKNL', match=filter_all + )), + ('AVX512_KNM', dict( + val='-xKNM', match=filter_all + )), + ('AVX512_SKX', dict( + val='-xSKYLAKE-AVX512', match=filter_all + )), + ('AVX512_CLX', dict( + val='-xCASCADELAKE', match=filter_all + )), + ('AVX512_CNL', dict( + val='-xCANNONLAKE', match=filter_all + )), + ('AVX512_ICL', dict( + val='-xICELAKE-CLIENT', match=filter_all + )), + ): + features[fet].update_method(state, [], {'args': arg}) + elif 'msvc' in cid: + # SSE3, SSSE3, SSE41, POPCNT, SSE42, F16C, XOP and FMA4 + # don't own private FLAGS still MS's compiler provides + # ABI capability for them. + for fet, arg in ( + ('SSE', dict(val='/arch:SSE')), + ('SSE2', dict(val='/arch:SSE2', match='/arch:.*')), + ('AVX', dict(val='/arch:AVX', match='/arch:.*')), + ('AVX2', dict(val='/arch:AVX2', match='/arch:.*')), + ('FMA3', dict(val='/arch:AVX2', match='/arch:.*')), + ('AVX512_COMMON', dict(val='/arch:AVX512', match='/arch:.*')), + ('AVX512_SKX', dict(val='/arch:AVX512', match='/arch:.*')), + ): + features[fet].update_method(state, [], {'args': arg}) + # MSVC special headers + for fet, header in ( + ('POPCNT', 'nmmintrin.h'), + ('XOP', 'ammintrin.h'), + ('FMA4', 'ammintrin.h'), + ): + features[fet].update_method(state, [], {'headers': header}) + + # MSVC doesn't support FMA3 or AVX2 independently + # same for AVX512_COMMON and AVX512_SKX + for fet, implies in ( + 'FMA3', ('F16C', 'AVX2'), + 'AVX2', ('F16C', 'FMA3'), + 'AVX512_COMMON', ('FMA3', 'AVX2', 'AVX512_SKX'), + ): + features[fet].update_method(state, [], { + 'implies': [features[sub_fet] for sub_fet in implies] + }) + for dis_fet in ('AVX512_KNL', 'AVX512_KNM'): + features[dis_fet].update_method(state, [], { + 'disable': 'MSVC compiler does not support it' + }) + # unix-like (gcc, clang) + else: + for fet in ( + 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'POPCNT', + 'AVX', 'F16C', 'XOP', 'FMA4', 'AVX2' + ): + features[fet].update_method(state, [], { + 'args': '-m' + fet.lower() + }) + + for fet, args in ( + ('SSE41', '-msse4.1'), + ('SSE42', '-msse4.2'), + ('FMA3', '-mfma'), + # should we add "-mno-mmx"? + ('AVX512_COMMON', '-mavx512f -mavx512cd'), + ('AVX512_KNL', '-mavx512er -mavx512pf'), + ('AVX512_KNM', '-mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq'), + ('AVX512_SKX', '-mavx512vl -mavx512bw -mavx512dq'), + ('AVX512_CLX', '-mavx512vnni'), + ('AVX512_CNL', '-mavx512ifma -mavx512vbmi'), + ('AVX512_ICL', '-mavx512vbmi2 -mavx512bitalg -mavx512vpopcntdq') + ): + features[fet].update_method(state, [], { + 'args': args.split() + }) + + for fet in ( + 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'POPCNT', + 'AVX', 'F16C', 'XOP', 'FMA4', 'AVX2' + ): + features[fet].update_method(state, [], { + 'args': '-m' + fet.lower() + }) + +def x86_features(state: 'ModuleState', compiler: 'Compiler' + ) -> T.Dict[str, FeatureObject]: + features: T.Dict[str, FeatureObject] = _init_features(state) + _features_args(state, compiler, features) + return features From 18db1ffd4b0e9a97ead6584b45589b29782659c1 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 18 Jan 2023 03:51:56 +0200 Subject: [PATCH 03/25] Fix runtime detect of AVX features --- mesonbuild/modules/feature/x86_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/modules/feature/x86_features.py b/mesonbuild/modules/feature/x86_features.py index c5eadad1bf85..376ca6a83281 100644 --- a/mesonbuild/modules/feature/x86_features.py +++ b/mesonbuild/modules/feature/x86_features.py @@ -159,7 +159,7 @@ def _init_features(state: 'ModuleState') -> T.Dict[str, FeatureObject]: AVX = fet('AVX', 20, dict( implies=SSE42, headers='immintrin.h', - detect=dict(val='AVX', match='.*'), + detect=dict(val='AVX', match='.*SSE.*'), test_code='''\ __m256 s0 = _mm256_loadu_ps((float*)src); __m256 s1 = _mm256_loadu_ps((float*)src+8); From 64b715261ecee003faf9a38d2cf1da49182bbea3 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 18 Jan 2023 08:29:58 +0200 Subject: [PATCH 04/25] Satisfy the linter --- mesonbuild/modules/feature/feature.py | 107 ++++++++++++--------- mesonbuild/modules/feature/module.py | 36 +++---- mesonbuild/modules/feature/x86_features.py | 8 +- 3 files changed, 76 insertions(+), 75 deletions(-) diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/feature/feature.py index 3767445dd449..d0166fc69983 100644 --- a/mesonbuild/modules/feature/feature.py +++ b/mesonbuild/modules/feature/feature.py @@ -187,7 +187,7 @@ def _implattr(opt_name: str, values: 'IMPLIED_ATTR', if not isinstance(edict, dict): # It shouldn't happen continue - unknown_keys = tuple([k for k in edict.keys() if k not in accepted_keys]) + unknown_keys = [k for k in edict.keys() if k not in accepted_keys] if unknown_keys: raise MesonException( f'feature.new: unknown keys {unknown_keys} in ' @@ -222,6 +222,7 @@ def implattr(opt_name: str) -> T.Callable[ T.Union[None, T.List[ImpliedAttr]]]: return lambda values: Convert._implattr(opt_name, values) + if T.TYPE_CHECKING: class FeatureKwArgs(TypedDict): #implies: T.Optional[T.List['FeatureObject']] @@ -258,34 +259,33 @@ def __init__(self, state: 'ModuleState', str, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, (dict, str)), ) + @typed_pos_args('feature.new', str, int) @typed_kwargs('feature.new', - KwargInfo('implies', + KwargInfo( + 'implies', (FeatureObject, ContainerTypeInfo(list, FeatureObject)), default=[], listify=True ), - KwargInfo('group', - (str, ContainerTypeInfo(list, str)), + KwargInfo( + 'group', (str, ContainerTypeInfo(list, str)), default=[], listify=True ), - KwargInfo('detect', - IMPLIED_ATTR_TYPES, convertor=Convert.implattr('detect'), - default=[] - ), - KwargInfo('args', - IMPLIED_ATTR_TYPES, convertor=Convert.implattr('args'), - default=[] + KwargInfo( + 'detect', IMPLIED_ATTR_TYPES, + convertor=Convert.implattr('detect'), default=[] ), - KwargInfo('headers', - IMPLIED_ATTR_TYPES, convertor=Convert.implattr('headers'), + KwargInfo( + 'args', IMPLIED_ATTR_TYPES, convertor=Convert.implattr('args'), default=[] ), KwargInfo( - 'test_code', (str, File), - default='' + 'headers', IMPLIED_ATTR_TYPES, + convertor=Convert.implattr('headers'), default=[] ), - KwargInfo('extra_tests', - (ContainerTypeInfo(dict, (str, File))), + KwargInfo('test_code', (str, File), default=''), + KwargInfo( + 'extra_tests', (ContainerTypeInfo(dict, (str, File))), default={} ), KwargInfo('disable', (str), default=''), @@ -299,7 +299,7 @@ def init_attrs(state: 'ModuleState', self.implies = set(kwargs['implies']) self.group = kwargs['group'] self.detect = kwargs['detect'] - self.args = kwargs['args'] + self.args = kwargs['args'] self.headers = kwargs['headers'] self.test_code = kwargs['test_code'] self.extra_tests = kwargs['extra_tests'] @@ -345,29 +345,38 @@ def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], NoneType, str, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, (dict, str)), ) + @noPosargs @typed_kwargs('feature.update', KwargInfo('name', (NoneType, str)), KwargInfo('interest', (NoneType, int)), - KwargInfo('implies', - (NoneType, FeatureObject, ContainerTypeInfo(list, FeatureObject)), + KwargInfo( + 'implies', ( + NoneType, FeatureObject, + ContainerTypeInfo(list, FeatureObject) + ), listify=True ), - KwargInfo('group', - (NoneType, str, ContainerTypeInfo(list, str)), listify=True + KwargInfo( + 'group', (NoneType, str, ContainerTypeInfo(list, str)), + listify=True ), - KwargInfo('detect', - IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('detect') + KwargInfo( + 'detect', IMPLIED_ATTR_NTYPES, + convertor=Convert.implattr('detect') ), - KwargInfo('args', - IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('args') + KwargInfo( + 'args', IMPLIED_ATTR_NTYPES, + convertor=Convert.implattr('args') ), - KwargInfo('headers', - IMPLIED_ATTR_NTYPES, convertor=Convert.implattr('headers') + KwargInfo( + 'headers', IMPLIED_ATTR_NTYPES, + convertor=Convert.implattr('headers') ), KwargInfo('test_code', (NoneType, str, File)), - KwargInfo('extra_tests', - (NoneType, ContainerTypeInfo(dict, (str, File))) + KwargInfo( + 'extra_tests', ( + NoneType, ContainerTypeInfo(dict, (str, File))) ), KwargInfo('disable', (NoneType, str)), ) @@ -397,11 +406,11 @@ def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str]]]: ret[attr] = [v.to_dict() for v in getattr(self, attr)] return ret - def get_implicit(self, _caller: 'T.Set[FeatureObject]' = None - ) -> 'T.Set[FeatureObject]': + def get_implicit(self, _caller: T.Set['FeatureObject'] = None + ) -> T.Set['FeatureObject']: # infinity recursive guard since # features can imply each other - _caller = {self,} if not _caller else _caller.union({self,}) + _caller = {self, } if not _caller else _caller.union({self, }) implies = self.implies.difference(_caller) ret = self.implies for sub_fet in implies: @@ -409,7 +418,7 @@ def get_implicit(self, _caller: 'T.Set[FeatureObject]' = None return ret def test_args(self, state: 'ModuleState', compiler: 'Compiler' - ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: + ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: check_args = compiler.has_multi_arguments args: T.List[ImpliedAttr] = [] skipped_args: T.List[ImpliedAttr] = [] @@ -424,8 +433,9 @@ def test_args(self, state: 'ModuleState', compiler: 'Compiler' return cached, args, skipped_args def test_headers(self, state: 'ModuleState', compiler: 'Compiler', - args: T.List[str] - ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: + args: T.List[str]) -> T.Tuple[ + bool, T.List[ImpliedAttr], T.List[ImpliedAttr] + ]: check_header = compiler.check_header headers: T.List[ImpliedAttr] = [] skipped_headers: T.List[ImpliedAttr] = [] @@ -442,11 +452,9 @@ def test_headers(self, state: 'ModuleState', compiler: 'Compiler', cached &= tcached return cached, headers, skipped_headers - def test_extra(self, state: 'ModuleState', - compiler: 'Compiler', - args: T.List[str], - headers: T.List[str] - ) -> T.Tuple[bool, T.List[str], T.List[str]]: + def test_extra(self, state: 'ModuleState', compiler: 'Compiler', + args: T.List[str], headers: T.List[str] + ) -> T.Tuple[bool, T.List[str], T.List[str]]: extra: T.List[str] = [] skipped_extra: T.List[str] = [] cached = True @@ -505,13 +513,13 @@ def test_impl(self, state: 'ModuleState', compiler: 'Compiler', if self.disable: return False, True, f'{prefix}disabled due to {self.disable}', None - _caller = {self,} if not _caller else _caller.union({self,}) + _caller = {self, } if not _caller else _caller.union({self, }) cached = True result = TestResult(implicit=self.implies.difference(_caller)) after = TestResult() implicit: T.List[FeatureObject] = sorted(result.implicit) for fet in implicit: - imp_ret = fet.test_impl(state, compiler, force_args, _caller) + imp_ret = fet.test_impl(state, compiler, force_args, _caller) imp_cached, _, _, imp_result = imp_ret if not imp_result: return imp_ret @@ -526,8 +534,8 @@ def test_impl(self, state: 'ModuleState', compiler: 'Compiler', if (not rargs and skipped_args) and not self.test_code: return ( tcached, False, - f'{prefix}specified arguments {tuple(skipped_args)} are not ' - 'supported and the test file is not specified', + f'{prefix}specified arguments {tuple(skipped_args)} ' + 'are not supported and the test file is not specified', None ) result.args = ImpliedAttr.normalize( @@ -538,12 +546,15 @@ def test_impl(self, state: 'ModuleState', compiler: 'Compiler', else: args = force_args - tcached, rheaders, skipped_headers = self.test_headers(state, compiler, args) + tcached, rheaders, skipped_headers = self.test_headers( + state, compiler, args + ) if not rheaders and skipped_headers: return ( tcached, False, - f'{prefix}specified headers {tuple(h.val for h in skipped_headers)} ' - 'are not supported by the compiler', + f'{prefix}specified headers ' + f'{[h.val for h in skipped_headers]}' + 'are not supported by the compiler', None ) cached &= tcached diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py index 243f74ffe388..34814d4ea6f9 100644 --- a/mesonbuild/modules/feature/module.py +++ b/mesonbuild/modules/feature/module.py @@ -94,20 +94,14 @@ def cpu_features_method(self, state: 'ModuleState', features.update(func(state, compiler)) return features - @typed_pos_args('feature.test', - varargs=FeatureObject - ) + @typed_pos_args('feature.test', varargs=FeatureObject) @typed_kwargs('feature.test', - KwargInfo('compiler', - (NoneType, Compiler) - ), - KwargInfo('force_args', - (NoneType, str, ContainerTypeInfo(list, str)), + KwargInfo('compiler', (NoneType, Compiler)), + KwargInfo( + 'force_args', (NoneType, str, ContainerTypeInfo(list, str)), listify=True ), - KwargInfo('any', - bool, default=True - ) + KwargInfo('any', bool, default=True) ) def test_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -141,14 +135,12 @@ def test_method(self, state: 'ModuleState', return [False, {}] return [True, result.to_dict()] - @typed_pos_args('feature.implicit', - varargs=FeatureObject - ) + @typed_pos_args('feature.implicit', varargs=FeatureObject) @noKwargs def implicit_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: features = args[0] implicit = set().union(*[fet.get_implicit() for fet in features]) @@ -156,9 +148,7 @@ def implicit_method(self, state: 'ModuleState', implicit.difference_update(set(features)) return sorted(implicit) - @typed_pos_args('feature.ahead', - varargs=FeatureObject - ) + @typed_pos_args('feature.ahead', varargs=FeatureObject) @noKwargs def ahead_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -174,9 +164,7 @@ def ahead_method(self, state: 'ModuleState', ahead = list(sorted(features, reverse=True)[:1]) return ahead - @typed_pos_args('feature.untied', - varargs=FeatureObject - ) + @typed_pos_args('feature.untied', varargs=FeatureObject) @noKwargs def untied_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -190,7 +178,7 @@ def untied_method(self, state: 'ModuleState', if sub_fet in fet.implies and fet in sub_fet.implies } if tied: - stied = sorted(tied.union({fet,})) + stied = sorted(tied.union({fet, })) if fet not in stied[1:]: continue ret.remove(stied[0]) diff --git a/mesonbuild/modules/feature/x86_features.py b/mesonbuild/modules/feature/x86_features.py index 376ca6a83281..5d943b202021 100644 --- a/mesonbuild/modules/feature/x86_features.py +++ b/mesonbuild/modules/feature/x86_features.py @@ -320,7 +320,9 @@ def _init_features(state: 'ModuleState') -> T.Dict[str, FeatureObject]: m64 = _kxor_mask64(m64, m64); m64 = _cvtu64_mask64(_cvtmask64_u64(m64)); m64 = _mm512_kunpackd(m64, m64); - m64 = (__mmask64)_mm512_kunpackw((__mmask32)m64, (__mmask32)m64); + m64 = (__mmask64)_mm512_kunpackw( + (__mmask32)m64, (__mmask32)m64 + ); *(int*)src = (int)_cvtmask64_u64(m64); ''', AVX512DQ_MASK=f'''\ @@ -445,14 +447,14 @@ def _features_args(state: 'ModuleState', # we specify the arguments that we need to filterd # from implied features rather than blinedly removes all args(.*) # since args may updated from outside. - filter_all = ".*m[sse|avx|arch\=|-x[a-z0-9\-]].*" + filter_all = r'.*m[sse|avx|arch\=|-x[a-z0-9\-]].*' for fet, arg in ( ('SSE41', dict(val='-msse4.1')), ('SSE42', dict(val='-msse4.2')), ('FMA3', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), ('AVX2', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), ('AVX512_COMMON', dict( - val='-march=common-avx512', match='.*m[sse|avx|arch\=].*' + val='-march=common-avx512', match=r'.*m[sse|avx|arch\=].*' )), ('AVX512_KNL', dict( val='-xKNL', match=filter_all From 5a1de8b4eba18f17373b56f5bb374eb5314dd33d Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 25 Jan 2023 10:09:44 +0200 Subject: [PATCH 05/25] replace method to_dict with get --- mesonbuild/modules/feature/feature.py | 58 ++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/feature/feature.py index d0166fc69983..f5a626750b86 100644 --- a/mesonbuild/modules/feature/feature.py +++ b/mesonbuild/modules/feature/feature.py @@ -39,7 +39,7 @@ from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, - ContainerTypeInfo, + ContainerTypeInfo, noArgsFlattening ) from .. import ModuleObject from .utils import test_code, get_compiler @@ -123,8 +123,6 @@ def normalize(lst: 'T.List[ImpliedAttr]' @dataclass class TestResult: - #implicit: T.Set['FeatureObject'] = field(default_factory=set) - implicit: T.Set[T.Any] = field(default_factory=set) args: T.List[ImpliedAttr] = field(default_factory=list) detect: T.List[ImpliedAttr] = field(default_factory=list) headers: T.List[ImpliedAttr] = field(default_factory=list) @@ -136,7 +134,6 @@ class TestResult: def __add__(self, robj: 'TestResult') -> 'TestResult': return TestResult( - implicit = self.implicit.union(robj.implicit), args = ImpliedAttr.normalize(self.args + robj.args), headers = ImpliedAttr.normalize(self.headers + robj.headers), detect = ImpliedAttr.normalize(self.detect + robj.detect), @@ -159,7 +156,6 @@ def to_dict(self) -> T.Dict[str, T.List[str]]: for attr in ('defines', 'undefines'): ret[attr] = getattr(self, attr)[:] - ret['implicit'] = [str(fet) for fet in sorted(self.implicit)] ret['support'] = self.support.to_list() return ret @@ -255,6 +251,8 @@ def __init__(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> None: + super().__init__() + IMPLIED_ATTR_TYPES = ( str, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, (dict, str)), @@ -305,11 +303,10 @@ def init_attrs(state: 'ModuleState', self.extra_tests = kwargs['extra_tests'] self.disable: str = kwargs['disable'] - super().__init__() init_attrs(state, args, kwargs) self.methods.update({ 'update': self.update_method, - 'to_dict': self.to_dict_method, + 'get': self.get_method, }) def __hash__(self) -> int: @@ -336,9 +333,6 @@ def __gt__(self, robj: object) -> T.Any: def __ge__(self, robj: object) -> T.Any: return robj <= self - def __str__(self) -> str: - return self.name - def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'FeatureObject': IMPLIED_ATTR_NTYPES = ( @@ -347,7 +341,7 @@ def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], ) @noPosargs - @typed_kwargs('feature.update', + @typed_kwargs('feature.feature.update', KwargInfo('name', (NoneType, str)), KwargInfo('interest', (NoneType, int)), KwargInfo( @@ -391,20 +385,30 @@ def update(state: 'ModuleState', args: T.List['TYPE_var'], update(state, args, kwargs) return self - @noPosargs @noKwargs - def to_dict_method(self, state: 'ModuleState', args: T.List['TYPE_var'], - kwargs: 'TYPE_kwargs' - ) -> T.Dict[str, T.Union[str, T.List[str]]]: - return self.to_dict() - - def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str]]]: - ret = self.__dict__.copy() - ret.pop('methods') - ret['implies'] = [str(fet) for fet in sorted(self.implies)] - for attr in ('detect', 'args', 'headers'): - ret[attr] = [v.to_dict() for v in getattr(self, attr)] - return ret + @typed_pos_args('feature.feature.get', str) + def get_method(self, state: 'ModuleState', args: T.Tuple[str], + kwargs: 'TYPE_kwargs') -> 'TYPE_var': + + impl_lst = lambda lst: [v.to_dict() for v in lst] + noconv = lambda v: v + dfunc = dict( + name = noconv, + interest = noconv, + group = noconv, + implies = lambda v: [fet.name for fet in sorted(v)], + detect = impl_lst, + args = impl_lst, + headers = impl_lst, + test_code = noconv, + extra_tests = noconv, + disable = noconv + ) + cfunc = dfunc.get(args[0]) + if cfunc is None: + raise MesonException(f'Key {args[0]!r} is not in the feature.') + + return cfunc(getattr(self, args[0])) def get_implicit(self, _caller: T.Set['FeatureObject'] = None ) -> T.Set['FeatureObject']: @@ -515,10 +519,10 @@ def test_impl(self, state: 'ModuleState', compiler: 'Compiler', _caller = {self, } if not _caller else _caller.union({self, }) cached = True - result = TestResult(implicit=self.implies.difference(_caller)) + result = TestResult() after = TestResult() - implicit: T.List[FeatureObject] = sorted(result.implicit) - for fet in implicit: + implies: T.List[FeatureObject] = sorted(self.implies.difference(_caller)) + for fet in implies: imp_ret = fet.test_impl(state, compiler, force_args, _caller) imp_cached, _, _, imp_result = imp_ret if not imp_result: From 6c90fa5a0c2421ddcbaf5e52a3955707202524c9 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 25 Jan 2023 10:10:42 +0200 Subject: [PATCH 06/25] improve test method --- mesonbuild/modules/feature/module.py | 91 ++++++++++++++++++---------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py index 34814d4ea6f9..4afefd577f38 100644 --- a/mesonbuild/modules/feature/module.py +++ b/mesonbuild/modules/feature/module.py @@ -66,6 +66,7 @@ def __init__(self) -> None: 'implicit': self.implicit_method, 'ahead': self.ahead_method, 'untied': self.untied_method, + 'sort': self.sort_method, }) def new_method(self, state: 'ModuleState', @@ -94,14 +95,14 @@ def cpu_features_method(self, state: 'ModuleState', features.update(func(state, compiler)) return features - @typed_pos_args('feature.test', varargs=FeatureObject) + @typed_pos_args('feature.test', varargs=FeatureObject, min_varargs=1) @typed_kwargs('feature.test', KwargInfo('compiler', (NoneType, Compiler)), KwargInfo( 'force_args', (NoneType, str, ContainerTypeInfo(list, str)), listify=True ), - KwargInfo('any', bool, default=True) + KwargInfo('any', bool, default=False) ) def test_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -115,27 +116,34 @@ def test_method(self, state: 'ModuleState', if not compiler: compiler = get_compiler(state) - features = args[0] - if len(features) == 1: - result = features[0].test(state, compiler, force_args) - if result is None: - return [False, {}] - return [True, result.to_dict()] + features = self.ahead(args[0]) + if any_: + features = self.implicit_c(features) - features = sorted(features) result = features[0].test(state, compiler, force_args) - for fet in features[1:]: - ret = fet.test(state, compiler, force_args) - if not ret: - if not any_: - return None - continue - result += ret + supported_features = [] + if result is not None: + supported_features = [features[0]] + for fet in features[1:]: + ret = fet.test(state, compiler, force_args) + if ret is None: + if any_: + continue + result = ret + break + supported_features += [fet] + result += ret + if result is None: return [False, {}] - return [True, result.to_dict()] - @typed_pos_args('feature.implicit', varargs=FeatureObject) + result = result.to_dict() + result['features'] = [ + fet.name for fet in self.implicit_c(supported_features) + ] + return [True, result] + + @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) @noKwargs def implicit_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -143,12 +151,9 @@ def implicit_method(self, state: 'ModuleState', ) -> T.List[FeatureObject]: features = args[0] - implicit = set().union(*[fet.get_implicit() for fet in features]) - # since features can imply each other - implicit.difference_update(set(features)) - return sorted(implicit) + return self.implicit(features) - @typed_pos_args('feature.ahead', varargs=FeatureObject) + @typed_pos_args('feature.ahead', varargs=FeatureObject, min_varargs=1) @noKwargs def ahead_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -156,15 +161,9 @@ def ahead_method(self, state: 'ModuleState', ) -> T.List[FeatureObject]: features = args[0] - implicit = set().union(*[fet.get_implicit() for fet in features]) - ahead = [fet for fet in features if fet not in implicit] - if len(ahead) == 0: - # return the highest interested feature - # if all features imply each other - ahead = list(sorted(features, reverse=True)[:1]) - return ahead + return self.ahead(features) - @typed_pos_args('feature.untied', varargs=FeatureObject) + @typed_pos_args('feature.untied', varargs=FeatureObject, min_varargs=1) @noKwargs def untied_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], @@ -185,3 +184,33 @@ def untied_method(self, state: 'ModuleState', ret.append(fet) return ret + @typed_pos_args('feature.sort', varargs=FeatureObject, min_varargs=1) + @noKwargs + def sort_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + return sorted(args[0]) + + @staticmethod + def implicit(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: + implicit = set().union(*[fet.get_implicit() for fet in features]) + # since features can imply each other + implicit.difference_update(set(features)) + return sorted(implicit) + + @staticmethod + def implicit_c(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: + implicit = set().union(*[fet.get_implicit() for fet in features], features) + return sorted(implicit) + + @staticmethod + def ahead(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: + implicit = set().union(*[fet.get_implicit() for fet in features]) + ahead = [fet for fet in features if fet not in implicit] + if len(ahead) == 0: + # return the highest interested feature + # if all features imply each other + ahead = list(sorted(features, reverse=True)[:1]) + return ahead + From 54fe62dad1209016d0b7d5b19023dc45026018b1 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 26 Jan 2023 04:56:33 +0200 Subject: [PATCH 07/25] Fix returned boolean value when test fails --- mesonbuild/modules/feature/feature.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/feature/feature.py index f5a626750b86..69d13abc8d4f 100644 --- a/mesonbuild/modules/feature/feature.py +++ b/mesonbuild/modules/feature/feature.py @@ -475,7 +475,7 @@ def test_extra(self, state: 'ModuleState', compiler: 'Compiler', def test(self, state: 'ModuleState', compiler: 'Compiler', force_args: T.Optional[T.List[str]] = None - ) -> TestResult: + ) -> T.Optional[TestResult]: cached, disabled, error, result = self.test_impl( state, compiler, force_args @@ -498,7 +498,7 @@ def test(self, state: 'ModuleState', compiler: 'Compiler', 'due to', error ) - return TestResult(support=FeatureSupport.NONE) + return None mlog.log( log_prefix, mlog.green('Supported'), From f18398f9a6ab065f6f645d05c4655ce68acab127 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 12 Jul 2023 20:12:42 +0400 Subject: [PATCH 08/25] Shrink the module - Removes CPU features implmentation and move them NumPy meson scripts - Removes attr 'header', since it can be handeled within a configuration file - re-implment method test, make it more simple - Implment method `multi_targets` Handels multi_targets via meson script was pretty anoyning from python make it much simpler --- mesonbuild/modules/feature/feature.py | 499 ++++--------------- mesonbuild/modules/feature/module.py | 533 +++++++++++++++----- mesonbuild/modules/feature/utils.py | 35 +- mesonbuild/modules/feature/x86_features.py | 554 --------------------- 4 files changed, 503 insertions(+), 1118 deletions(-) delete mode 100644 mesonbuild/modules/feature/x86_features.py diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/feature/feature.py index 69d13abc8d4f..e269d6231270 100644 --- a/mesonbuild/modules/feature/feature.py +++ b/mesonbuild/modules/feature/feature.py @@ -1,40 +1,11 @@ # Copyright (c) 2023, NumPy Developers. # All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the NumPy Developers nor the names of any -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import typing as T import re from dataclasses import dataclass, field from enum import IntFlag, auto -from ... import mlog from ...mesonlib import File, MesonException from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( @@ -51,22 +22,25 @@ from ...compilers import Compiler from .. import ModuleState -class FeatureSupport(IntFlag): - NONE: int = 0 - ARG: int = auto() - FILE: int = auto() - - def __str__(self) -> str: - return ', '.join(self.to_list()) - - def to_list(self) -> T.List[str]: - return [ - attr for attr in ('ARG', 'FILE') - if getattr(FeatureSupport, attr) in self - ] - @dataclass(unsafe_hash=True, order=True) -class ImpliedAttr: +class ConflictAttr: + """ + Data class representing an feature attribute that may conflict + with other features attributes. + + The reason behind this class to clear any possible conflicts with + compiler arguments when they joined together due gathering + the implied features or concatenate non-implied features. + + Attributes: + val: The value of the feature attribute. + match: Regular expression pattern for matching conflicted values + (optional). + mfilter: Regular expression pattern for filtering these conflicted values + (optional). + mjoin: String used to join filtered values (optional) + + """ val: str = field(hash=True, compare=True) match: T.Union[re.Pattern, None] = field( default=None, hash=False, compare=False @@ -79,8 +53,8 @@ class ImpliedAttr: def __str__(self) -> str: return self.val - def copy(self) -> 'ImpliedAttr': - return ImpliedAttr(**self.__dict__) + def copy(self) -> 'ConflictAttr': + return ConflictAttr(**self.__dict__) def to_dict(self) -> T.Dict[str, str]: ret: T.Dict[str, str] = {} @@ -95,107 +69,52 @@ def to_dict(self) -> T.Dict[str, str]: ret[attr] = val return ret - @staticmethod - def normalize(lst: 'T.List[ImpliedAttr]' - ) -> 'T.List[ImpliedAttr]': - ret: T.List[ImpliedAttr] = [] - for attr in lst: - if not attr.match: - if attr not in ret: - ret.append(attr) - continue - new_ret: T.List[ImpliedAttr] = [] - attr = attr.copy() - for ret_attr in ret: - if not attr.match.match(ret_attr.val): - new_ret.append(ret_attr) - continue - if not attr.mfilter: - continue - val = attr.mfilter.findall(ret_attr.val) - if not val: - continue - attr.val += ret_attr.mjoin.join(val) - if attr not in ret: - new_ret.append(attr) - ret = new_ret - return ret - -@dataclass -class TestResult: - args: T.List[ImpliedAttr] = field(default_factory=list) - detect: T.List[ImpliedAttr] = field(default_factory=list) - headers: T.List[ImpliedAttr] = field(default_factory=list) - defines: T.List[str] = field(default_factory=list) - undefines: T.List[str] = field(default_factory=list) - support: FeatureSupport = field( - default=FeatureSupport.ARG | FeatureSupport.FILE - ) - - def __add__(self, robj: 'TestResult') -> 'TestResult': - return TestResult( - args = ImpliedAttr.normalize(self.args + robj.args), - headers = ImpliedAttr.normalize(self.headers + robj.headers), - detect = ImpliedAttr.normalize(self.detect + robj.detect), - defines = self.defines + [ - v for v in robj.defines - if v not in self.defines - ], - undefines = self.undefines + [ - v for v in robj.undefines - if v not in self.undefines - ], - support = self.support & robj.support - ) - - def to_dict(self) -> T.Dict[str, T.List[str]]: - ret = {} - for attr in ('args', 'detect', 'headers'): - ret[attr] = [str(v) for v in getattr(self, attr)] - - for attr in ('defines', 'undefines'): - ret[attr] = getattr(self, attr)[:] - - ret['support'] = self.support.to_list() - return ret - - -if T.TYPE_CHECKING: - IMPLIED_ATTR = T.Union[ - None, str, T.Dict[str, str], T.List[ - T.Union[str, T.Dict[str, str]] +class KwargConfilctAttr(KwargInfo): + def __init__(self, func_name: str, opt_name: str, default: T.Any = None): + types = [ + str, ContainerTypeInfo(dict, str), + ContainerTypeInfo(list, (dict, str)) ] - ] + if default is None: + types += [NoneType] + super().__init__( + opt_name, tuple(types), + convertor = lambda values: self.convert( + func_name, opt_name, values + ), + default = default + ) -class Convert: @staticmethod - def _implattr(opt_name: str, values: 'IMPLIED_ATTR', - ) -> T.Union[None, T.List[ImpliedAttr]]: + def convert(func_name:str, opt_name: str, values: 'IMPLIED_ATTR', + ) -> T.Union[None, T.List[ConflictAttr]]: if values is None: return None - ret: T.List[ImpliedAttr] = [] + ret: T.List[ConflictAttr] = [] values = [values] if isinstance(values, (str, dict)) else values accepted_keys = ('val', 'match', 'mfilter', 'mjoin') for edict in values: if isinstance(edict, str): - ret.append(ImpliedAttr(val=edict)) + if edict: + ret.append(ConflictAttr(val=edict)) continue if not isinstance(edict, dict): # It shouldn't happen + # TODO: need exception here continue unknown_keys = [k for k in edict.keys() if k not in accepted_keys] if unknown_keys: raise MesonException( - f'feature.new: unknown keys {unknown_keys} in ' + f'{func_name}: unknown keys {unknown_keys} in ' f'option {opt_name}' ) val = edict.get('val') if val is None: raise MesonException( - f'feature.new: option "{opt_name}" requires ' + f'{func_name}: option "{opt_name}" requires ' f'a dictionary with key "val" to be set' ) - implattr = ImpliedAttr(val=val, mjoin=edict.get('mjoin', '')) + implattr = ConflictAttr(val=val, mjoin=edict.get('mjoin', '')) for cattr in ('match', 'mfilter'): cval = edict.get(cattr) if not cval: @@ -204,7 +123,7 @@ def _implattr(opt_name: str, values: 'IMPLIED_ATTR', ccval = re.compile(cval) except Exception as e: raise MesonException( - 'feature.new: unable to ' + '{func_name}: unable to ' f'compile the regex in option "{opt_name}"\n' f'"{cattr}:{cval}" -> {str(e)}' ) @@ -212,21 +131,18 @@ def _implattr(opt_name: str, values: 'IMPLIED_ATTR', ret.append(implattr) return ret - @staticmethod - def implattr(opt_name: str) -> T.Callable[ - ['IMPLIED_ATTR'], - T.Union[None, T.List[ImpliedAttr]]]: - return lambda values: Convert._implattr(opt_name, values) - - if T.TYPE_CHECKING: + IMPLIED_ATTR = T.Union[ + None, str, T.Dict[str, str], T.List[ + T.Union[str, T.Dict[str, str]] + ] + ] class FeatureKwArgs(TypedDict): #implies: T.Optional[T.List['FeatureObject']] implies: NotRequired[T.List[T.Any]] group: NotRequired[T.List[str]] - detect: NotRequired[T.List[ImpliedAttr]] - args: NotRequired[T.List[ImpliedAttr]] - headers: NotRequired[T.List[ImpliedAttr]] + detect: NotRequired[T.List[ConflictAttr]] + args: NotRequired[T.List[ConflictAttr]] test_code: NotRequired[T.Union[str, File]] extra_tests: NotRequired[T.Dict[str, T.Union[str, File]]] disable: NotRequired[str] @@ -236,13 +152,35 @@ class FeatureUpdateKwArgs(FeatureKwArgs): interest: NotRequired[int] class FeatureObject(ModuleObject): + """ + A data class that represents the feature. + + A feature is a unit of work that can be developed, tested, and deployed independently. + + Attributes: + name: The name of the feature. + interest: The interest level of the feature. + It used for sorting and to determine succor features. + implies: A set of features objects that are implied by this feature. + (Optional) + This means that if this feature is enabled, + then the implied features will also be enabled. + If any of the implied features is not supported by the platform + or the compiler this feature will also considerd not supported. + group: A list of + + detect: A list of strings that identify the methods that can be used to detect whether the feature is supported. + args: A list of strings that identify the arguments that are required to enable this feature. + test_code: A string or a file object that contains the test code for this feature. + extra_tests: A dictionary that maps from the name of a test to the test code for that test. + disable: A string that specifies why this feature is disabled. + """ name: str interest: int implies: T.Set['FeatureObject'] group: T.List[str] - detect: T.List[ImpliedAttr] - args: T.List[ImpliedAttr] - headers: T.List[ImpliedAttr] + detect: T.List[ConflictAttr] + args: T.List[ConflictAttr] test_code: T.Union[str, File] extra_tests: T.Dict[str, T.Union[str, File]] disable: str @@ -253,11 +191,6 @@ def __init__(self, state: 'ModuleState', super().__init__() - IMPLIED_ATTR_TYPES = ( - str, ContainerTypeInfo(dict, str), - ContainerTypeInfo(list, (dict, str)), - ) - @typed_pos_args('feature.new', str, int) @typed_kwargs('feature.new', KwargInfo( @@ -269,18 +202,8 @@ def __init__(self, state: 'ModuleState', 'group', (str, ContainerTypeInfo(list, str)), default=[], listify=True ), - KwargInfo( - 'detect', IMPLIED_ATTR_TYPES, - convertor=Convert.implattr('detect'), default=[] - ), - KwargInfo( - 'args', IMPLIED_ATTR_TYPES, convertor=Convert.implattr('args'), - default=[] - ), - KwargInfo( - 'headers', IMPLIED_ATTR_TYPES, - convertor=Convert.implattr('headers'), default=[] - ), + KwargConfilctAttr('feature.new', 'detect', default=[]), + KwargConfilctAttr('feature.new', 'args', default=[]), KwargInfo('test_code', (str, File), default=''), KwargInfo( 'extra_tests', (ContainerTypeInfo(dict, (str, File))), @@ -290,18 +213,21 @@ def __init__(self, state: 'ModuleState', ) def init_attrs(state: 'ModuleState', args: T.Tuple[str, int], - kwargs: 'FeatureKwArgs' - ) -> None: + kwargs: 'FeatureKwArgs') -> None: self.name = args[0] self.interest = args[1] self.implies = set(kwargs['implies']) self.group = kwargs['group'] self.detect = kwargs['detect'] self.args = kwargs['args'] - self.headers = kwargs['headers'] self.test_code = kwargs['test_code'] self.extra_tests = kwargs['extra_tests'] self.disable: str = kwargs['disable'] + if not self.detect: + if self.group: + self.detect = [ConflictAttr(val=f) for f in self.group] + else: + self.detect = [ConflictAttr(val=self.name)] init_attrs(state, args, kwargs) self.methods.update({ @@ -309,39 +235,10 @@ def init_attrs(state: 'ModuleState', 'get': self.get_method, }) - def __hash__(self) -> int: - return hash(str(id(self)) + self.name) - - def __eq__(self, robj: object) -> bool: - if not isinstance(robj, FeatureObject): - return False - return self is robj and self.name == robj.name - - def __lt__(self, robj: object) -> T.Any: - if not isinstance(robj, FeatureObject): - return NotImplemented - return self.interest < robj.interest - - def __le__(self, robj: object) -> T.Any: - if not isinstance(robj, FeatureObject): - return NotImplemented - return self.interest <= robj.interest - - def __gt__(self, robj: object) -> T.Any: - return robj < self - - def __ge__(self, robj: object) -> T.Any: - return robj <= self - def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'FeatureObject': - IMPLIED_ATTR_NTYPES = ( - NoneType, str, ContainerTypeInfo(dict, str), - ContainerTypeInfo(list, (dict, str)), - ) - @noPosargs - @typed_kwargs('feature.feature.update', + @typed_kwargs('feature.update', KwargInfo('name', (NoneType, str)), KwargInfo('interest', (NoneType, int)), KwargInfo( @@ -355,18 +252,8 @@ def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], 'group', (NoneType, str, ContainerTypeInfo(list, str)), listify=True ), - KwargInfo( - 'detect', IMPLIED_ATTR_NTYPES, - convertor=Convert.implattr('detect') - ), - KwargInfo( - 'args', IMPLIED_ATTR_NTYPES, - convertor=Convert.implattr('args') - ), - KwargInfo( - 'headers', IMPLIED_ATTR_NTYPES, - convertor=Convert.implattr('headers') - ), + KwargConfilctAttr('feature.update', 'detect'), + KwargConfilctAttr('feature.update', 'args'), KwargInfo('test_code', (NoneType, str, File)), KwargInfo( 'extra_tests', ( @@ -386,7 +273,7 @@ def update(state: 'ModuleState', args: T.List['TYPE_var'], return self @noKwargs - @typed_pos_args('feature.feature.get', str) + @typed_pos_args('feature.get', str) def get_method(self, state: 'ModuleState', args: T.Tuple[str], kwargs: 'TYPE_kwargs') -> 'TYPE_var': @@ -399,7 +286,6 @@ def get_method(self, state: 'ModuleState', args: T.Tuple[str], implies = lambda v: [fet.name for fet in sorted(v)], detect = impl_lst, args = impl_lst, - headers = impl_lst, test_code = noconv, extra_tests = noconv, disable = noconv @@ -407,7 +293,6 @@ def get_method(self, state: 'ModuleState', args: T.Tuple[str], cfunc = dfunc.get(args[0]) if cfunc is None: raise MesonException(f'Key {args[0]!r} is not in the feature.') - return cfunc(getattr(self, args[0])) def get_implicit(self, _caller: T.Set['FeatureObject'] = None @@ -421,199 +306,27 @@ def get_implicit(self, _caller: T.Set['FeatureObject'] = None ret = ret.union(sub_fet.get_implicit(_caller)) return ret - def test_args(self, state: 'ModuleState', compiler: 'Compiler' - ) -> T.Tuple[bool, T.List[ImpliedAttr], T.List[ImpliedAttr]]: - check_args = compiler.has_multi_arguments - args: T.List[ImpliedAttr] = [] - skipped_args: T.List[ImpliedAttr] = [] - cached = True - for a in self.args: - result, tcached = check_args([a.val], state.environment) - if result: - args.append(a) - else: - skipped_args.append(a) - cached &= tcached - return cached, args, skipped_args - - def test_headers(self, state: 'ModuleState', compiler: 'Compiler', - args: T.List[str]) -> T.Tuple[ - bool, T.List[ImpliedAttr], T.List[ImpliedAttr] - ]: - check_header = compiler.check_header - headers: T.List[ImpliedAttr] = [] - skipped_headers: T.List[ImpliedAttr] = [] - cached = True - for h in self.headers: - result, tcached = check_header( - hname=h.val, prefix='', env=state.environment, - extra_args=args - ) - if result: - headers.append(h) - else: - skipped_headers.append(h) - cached &= tcached - return cached, headers, skipped_headers - - def test_extra(self, state: 'ModuleState', compiler: 'Compiler', - args: T.List[str], headers: T.List[str] - ) -> T.Tuple[bool, T.List[str], T.List[str]]: - extra: T.List[str] = [] - skipped_extra: T.List[str] = [] - cached = True - for extra_name, extra_test in self.extra_tests.items(): - tcached, test, stderr = test_code( - state, compiler, args, headers, extra_test - ) - if test: - extra.append(extra_name) - else: - skipped_extra.append(extra_name) - cached &= tcached - return cached, extra, skipped_extra - - def test(self, state: 'ModuleState', compiler: 'Compiler', - force_args: T.Optional[T.List[str]] = None - ) -> T.Optional[TestResult]: - - cached, disabled, error, result = self.test_impl( - state, compiler, force_args - ) - log_prefix = f'Test feature "{mlog.bold(self.name)}" :' - cached_msg = f'({mlog.blue("cached")})'if cached else '' - if not result: - reason = ( - mlog.yellow('Disabled') if disabled - else mlog.red('Unsupported') - ) - mlog.log( - log_prefix, - reason, - cached_msg - ) - mlog.debug( - log_prefix, - reason, - 'due to', - error - ) - return None - mlog.log( - log_prefix, - mlog.green('Supported'), - cached_msg - ) - return result - - def test_impl(self, state: 'ModuleState', compiler: 'Compiler', - force_args: T.Optional[T.List[str]] = None, - _caller: T.Optional[T.Set['FeatureObject']] = None, - ) -> T.Tuple[ - bool, bool, str, T.Optional[TestResult] - ]: - - prefix = f'implied feature "{self.name}" ' if _caller else '' - if self.disable: - return False, True, f'{prefix}disabled due to {self.disable}', None + def __hash__(self) -> int: + return hash(str(id(self)) + self.name) - _caller = {self, } if not _caller else _caller.union({self, }) - cached = True - result = TestResult() - after = TestResult() - implies: T.List[FeatureObject] = sorted(self.implies.difference(_caller)) - for fet in implies: - imp_ret = fet.test_impl(state, compiler, force_args, _caller) - imp_cached, _, _, imp_result = imp_ret - if not imp_result: - return imp_ret - if fet > self: - after += imp_result - else: - result += imp_result - cached &= imp_cached - - if force_args is None: - tcached, rargs, skipped_args = self.test_args(state, compiler) - if (not rargs and skipped_args) and not self.test_code: - return ( - tcached, False, - f'{prefix}specified arguments {tuple(skipped_args)} ' - 'are not supported and the test file is not specified', - None - ) - result.args = ImpliedAttr.normalize( - result.args + rargs + after.args - ) - cached &= tcached - args = [a.val for a in result.args] - else: - args = force_args - - tcached, rheaders, skipped_headers = self.test_headers( - state, compiler, args - ) - if not rheaders and skipped_headers: - return ( - tcached, False, - f'{prefix}specified headers ' - f'{[h.val for h in skipped_headers]}' - 'are not supported by the compiler', - None - ) - cached &= tcached - result.headers = ImpliedAttr.normalize( - result.headers + rheaders + after.headers - ) - headers = [h.val for h in result.headers] - - if self.test_code: - tcached, test, stderr = test_code( - state, compiler, args, headers, self.test_code - ) - if not test: - return ( - tcached, False, - f'{prefix}compiler was not able to compile the test code\n' - f'Arguments: {str(args)}\n' - f'Headers: {str(headers)}\n' - f':\n"{stderr}"', - None - ) - cached &= tcached + def __eq__(self, robj: object) -> bool: + if not isinstance(robj, FeatureObject): + return False + return self is robj and self.name == robj.name - if not self.detect: - detect = [ - ImpliedAttr(val=name) for name in ( - self.group if self.group else [self.name] - ) - ] - else: - detect = self.detect - result.detect = ImpliedAttr.normalize( - result.detect + detect + after.detect - ) + def __lt__(self, robj: object) -> T.Any: + if not isinstance(robj, FeatureObject): + return NotImplemented + return self.interest < robj.interest - tcached, extra, skipped_extra = self.test_extra( - state, compiler, args, headers - ) - cached &= tcached + def __le__(self, robj: object) -> T.Any: + if not isinstance(robj, FeatureObject): + return NotImplemented + return self.interest <= robj.interest - result.defines += [ - df for df in [self.name] + self.group + extra + after.defines - if df not in result.defines - ] - result.undefines += [ - udf for udf in skipped_extra + after.undefines - if udf not in result.undefines - ] + def __gt__(self, robj: object) -> T.Any: + return robj < self - support = FeatureSupport.NONE - if args and not skipped_args: - support = FeatureSupport.ARG - if self.test_code: - support |= FeatureSupport.FILE - support &= after.support + def __ge__(self, robj: object) -> T.Any: + return robj <= self - result.support &= support - return cached, False, '', result diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py index 4afefd577f38..ad56a1d59dc2 100644 --- a/mesonbuild/modules/feature/module.py +++ b/mesonbuild/modules/feature/module.py @@ -1,47 +1,20 @@ # Copyright (c) 2023, NumPy Developers. # All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the NumPy Developers nor the names of any -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import typing as T +import os +from ... import mlog, build from ...compilers import Compiler +from ...mesonlib import File, MesonException from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, - ContainerTypeInfo, + ContainerTypeInfo, permittedKwargs ) from .. import ModuleInfo, NewExtensionModule, ModuleReturnValue - -from .feature import FeatureObject -from .utils import get_compiler -from .x86_features import x86_features +from .feature import FeatureObject, ConflictAttr +from .utils import test_code, get_compiler if T.TYPE_CHECKING: from typing import TypedDict @@ -61,156 +34,442 @@ def __init__(self) -> None: super().__init__() self.methods.update({ 'new': self.new_method, - 'cpu_features': self.cpu_features_method, 'test': self.test_method, 'implicit': self.implicit_method, - 'ahead': self.ahead_method, - 'untied': self.untied_method, + 'implicit_c': self.implicit_c_method, 'sort': self.sort_method, + 'multi_target': self.multi_target_method, }) + # TODO: How to store and load from files in meson? + self.cached_tests = {} def new_method(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> FeatureObject: return FeatureObject(state, args, kwargs) - @noPosargs - @typed_kwargs('feature.cpu_features', - KwargInfo( - 'compiler', (NoneType, Compiler), - ), - ) - def cpu_features_method(self, state: 'ModuleState', - args: T.List['TYPE_var'], - kwargs: T.Dict[str, Compiler] - ) -> T.Dict[str, FeatureObject]: - compiler = kwargs['compiler'] - if compiler is None: - compiler = get_compiler(state) - - features: T.Dict[str, FeatureObject] = {} - for func in ( - x86_features, - ): - features.update(func(state, compiler)) - return features - @typed_pos_args('feature.test', varargs=FeatureObject, min_varargs=1) @typed_kwargs('feature.test', KwargInfo('compiler', (NoneType, Compiler)), + KwargInfo('anyfet', bool, default = False), KwargInfo( 'force_args', (NoneType, str, ContainerTypeInfo(list, str)), listify=True ), - KwargInfo('any', bool, default=False) ) def test_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], kwargs: 'TestKwArgs' ) -> T.List[T.Union[bool, T.Dict[str, T.Any]]]: + features = args[0] + features_set = set(features) + anyfet = kwargs['anyfet'] compiler = kwargs.get('compiler') - force_args = kwargs.get('force_args') - any_ = kwargs['any'] - if not compiler: compiler = get_compiler(state) - features = self.ahead(args[0]) - if any_: - features = self.implicit_c(features) + force_args = kwargs['force_args'] + if force_args is not None: + # removes in empty strings + force_args = [a for a in force_args if a] + + cached, test_result = self.test( + state, features=features_set, + compiler=compiler, + anyfet=anyfet, + force_args=force_args + ) + if not test_result['is_supported']: + if test_result['is_disabled']: + label = mlog.yellow('disabled') + else: + label = mlog.yellow('Unsupported') + else: + label = mlog.green('Supported') + if anyfet: + unsupported = [ + fet.name for fet in sorted(features_set) + if fet.name not in test_result['features'] + ] + if unsupported: + unsupported = ' '.join(unsupported) + label = mlog.green(f'Parial support, missing({unsupported})') - result = features[0].test(state, compiler, force_args) - supported_features = [] + features_names = ' '.join([f.name for f in features]) + log_prefix = f'Test features "{mlog.bold(features_names)}" :' + cached_msg = f'({mlog.blue("cached")})' if cached else '' + if not test_result['is_supported']: + mlog.log(log_prefix, label, 'due to', test_result['fail_reason']) + else: + mlog.log(log_prefix, label, cached_msg) + return [test_result['is_supported'], test_result] + + def test(self, state, features: T.Set[FeatureObject], + compiler: 'Compiler', + anyfet: bool = False, + force_args: T.Optional[T.Tuple[str]] = None, + _caller: T.Set[FeatureObject] = set() + ) -> T.Tuple[bool, T.Dict[str, T.Union[str, T.List[str]]]]: + # cached hash should inveolov all implied features + # since FeatureObject is mutable object. + implied_features = self.implicit(features) + test_hash = hash(( + tuple(sorted(features)), + tuple(sorted(implied_features)), + compiler, anyfet, + (-1 if force_args is None else tuple(force_args)) + )) + result = self.cached_tests.get(test_hash) if result is not None: - supported_features = [features[0]] - for fet in features[1:]: - ret = fet.test(state, compiler, force_args) - if ret is None: - if any_: + return True, result + + all_features = sorted(implied_features.union(features)) + if anyfet: + cached, test_result = self.test( + state, features=features, + compiler=compiler, + force_args=force_args + ) + if test_result['is_supported']: + self.cached_tests[test_hash] = test_result + return False, test_result + + features_any = set() + for fet in all_features: + _, test_result = self.test( + state, features={fet,}, + compiler=compiler, + force_args=force_args + ) + if test_result['is_supported']: + features_any.add(fet) + + _, test_result = self.test( + state, features=features_any, + compiler=compiler, + force_args=force_args + ) + self.cached_tests[test_hash] = test_result + return False, test_result + + # For multiple features, it important to erase any features + # implied by another to avoid duplicate testing since + # implied already tested also we use this set to genrate + # unque target name that can be used for multiple targets + # build. + prevalent_features = features.difference(implied_features) + if len(prevalent_features) == 0: + # It happens when all features imply each other. + # Set the highest interested feature + prevalent_features = sorted(features)[-1:] + else: + prevalent_features = sorted(prevalent_features) + + prevalent_names = [fet.name for fet in prevalent_features] + # prepare the result dict + test_result = { + 'target_name': '__'.join(prevalent_names), + 'prevalent_features': prevalent_names, + 'features': [fet.name for fet in all_features], + 'args': [], + 'detect': [], + 'defines': [], + 'undefines': [], + 'is_supported': True, + 'is_disabled': False, + 'fail_reason': '', + } + def fail_result(fail_reason, is_disabled = False, + result_dict = test_result.copy()): + result_dict.update({ + 'is_supported': False, + 'is_disabled': is_disabled, + 'fail_reason': fail_reason, + 'features': [] + }) + self.cached_tests[test_hash] = result_dict + return False, result_dict + + # since we allows features to imply each other + # items of `features` may part of `implied_features` + _caller = _caller.union(prevalent_features) + predecessor_features = implied_features.difference(_caller) + for fet in sorted(predecessor_features): + _, pred_result = self.test( + state, features={fet,}, + compiler=compiler, + force_args=force_args, + _caller=_caller + ) + if not pred_result['is_supported']: + reason = f'Implied feature "{fet.name}" ' + pred_disabled = pred_result['is_disabled'] + if pred_disabled: + fail_reason = reason + 'is disabled' + else: + fail_reason = reason + 'is not supported' + return fail_result(fail_reason, pred_disabled) + + for k in ['defines', 'undefines']: + values = test_result[k] + pred_values = pred_result[k] + values += [v for v in pred_values if v not in values] + + # Sort based on the lowest interest to deal with conflict attributes + # when combine all attributes togathers + conflict_attrs = ['detect'] + if force_args is None: + conflict_attrs += ['args'] + else: + test_result['args'] = force_args + + for fet in all_features: + for attr in conflict_attrs: + values: T.List[ConflictAttr] = getattr(fet, attr) + accumulate_values = test_result[attr] + for conflict in values: + conflict_val: str = conflict.val + if not conflict.match: + accumulate_values.append(conflict_val) continue - result = ret - break - supported_features += [fet] - result += ret + # select the acc items based on the match + new_acc: T.List[str] = [] + for acc in accumulate_values: + # not affected by the match so we keep it + if not conflict.match.match(acc): + new_acc.append(acc) + continue + # no filter so we totaly escape it + if not conflict.mfilter: + continue + filter_val = conflict.mfilter.findall(acc) + # no filter match so we totaly escape it + if not filter_val: + continue + conflict_val += conflict.mjoin.join(filter_val) + new_acc.append(conflict_val) + test_result[attr] = new_acc - if result is None: - return [False, {}] + test_args = compiler.has_multi_arguments + args = test_result['args'] + if args: + supported_args, test_cached = test_args(args, state.environment) + if not supported_args: + return fail_result( + f'Arguments "{", ".join(args)}" are not supported' + ) - result = result.to_dict() - result['features'] = [ - fet.name for fet in self.implicit_c(supported_features) - ] - return [True, result] + for fet in prevalent_features: + if fet.disable: + return fail_result( + f'{fet.name} is disabled due to "{fet.disable}"', + fet.disable + ) - @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) - @noKwargs - def implicit_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: + if fet.test_code: + _, tested_code, _ = test_code( + state, compiler, args, fet.test_code + ) + if not tested_code: + return fail_result( + f'Compiler fails against the test code of "{fet.name}"' + ) - features = args[0] - return self.implicit(features) + test_result['defines'] += [fet.name] + fet.group + for extra_name, extra_test in fet.extra_tests.items(): + _, tested_code, _ = test_code( + state, compiler, args, extra_test + ) + k = 'defines' if tested_code else 'undefines' + test_result[k].append(extra_name) - @typed_pos_args('feature.ahead', varargs=FeatureObject, min_varargs=1) - @noKwargs - def ahead_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: + self.cached_tests[test_hash] = test_result + return False, test_result - features = args[0] - return self.ahead(features) + @permittedKwargs(build.known_stlib_kwargs | { + 'dispatch', 'baseline', 'prefix' + }) + @typed_pos_args('feature.multi_target', str, varargs=( + str, File, build.CustomTarget, build.CustomTargetIndex, + build.GeneratedList, build.StructuredSources, build.ExtractedObjects, + build.BuildTarget + )) + @typed_kwargs('feature.multi_target', + KwargInfo( + 'dispatch', ContainerTypeInfo(list, + (FeatureObject, list) + ), + default=[] + ), + KwargInfo( + 'baseline', (NoneType, ContainerTypeInfo(list, FeatureObject)) + ), + KwargInfo('prefix', str, default=''), + KwargInfo('compiler', (NoneType, Compiler)), + allow_unknown=True + ) + def multi_target_method(self, state: 'ModuleState', + args: T.Tuple[str], kwargs: 'TYPE_kwargs' + ) -> T.List[T.Union[T.Dict[str, str], T.Any]]: + config_name = args[0] + sources = args[1] + dispatch = kwargs.pop('dispatch') + baseline = kwargs.pop('baseline') + prefix = kwargs.pop('prefix') + compiler = kwargs.pop('compiler') + if not compiler: + compiler = get_compiler(state) - @typed_pos_args('feature.untied', varargs=FeatureObject, min_varargs=1) - @noKwargs - def untied_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: - features = args[0] - ret: T.List[FeatureObject] = [] - for fet in features: - tied = { - sub_fet for sub_fet in ret - if sub_fet in fet.implies and fet in sub_fet.implies - } - if tied: - stied = sorted(tied.union({fet, })) - if fet not in stied[1:]: - continue - ret.remove(stied[0]) - ret.append(fet) - return ret + info = {} + if baseline is not None: + baseline_features = self.implicit_c(baseline) + cached, baseline = self.test( + state, features=set(baseline), anyfet=True, + compiler=compiler + ) + info['BASELINE'] = baseline + else: + baseline_features = [] + + dispatch_tests = [] + for d in dispatch: + if isinstance(d, FeatureObject): + target = {d,} + is_base_part = d in baseline_features + else: + target = set(d) + is_base_part = all([f in baseline_features for f in d]) + if is_base_part: + # TODO: add log + continue + cached, test_result = self.test( + state, features=target, + compiler=compiler + ) + if not test_result['is_supported']: + continue + target_name = test_result['target_name'] + if target_name in info: + continue + info[target_name] = test_result + dispatch_tests.append(test_result) + + dispatch_calls = [] + for test_result in dispatch_tests: + detect = '&&'.join([ + f'TEST_CB({d})' for d in test_result['detect'] + ]) + if detect: + detect = f'({detect})' + else: + detect = '1' + target_name = test_result['target_name'] + dispatch_calls.append( + f'{prefix}_MTARGETS_EXPAND(' + f'EXEC_CB({detect}, {target_name}, __VA_ARGS__)' + ')' + ) + + config_file = [ + '/* Autogenerated by the Meson features module. */', + '/* Do not edit, your changes will be lost. */', + '', + f'#undef {prefix}_MTARGETS_EXPAND', + f'#define {prefix}_MTARGETS_EXPAND(X) X', + '', + f'#undef {prefix}MTARGETS_CONF_BASELINE', + f'#define {prefix}MTARGETS_CONF_BASELINE(EXEC_CB, ...) ' + ( + f'{prefix}_MTARGETS_EXPAND(EXEC_CB(__VA_ARGS__))' + if baseline is not None + else '' + ), + '', + f'#undef {prefix}MTARGETS_CONF_DISPATCH', + f'#define {prefix}MTARGETS_CONF_DISPATCH(TEST_CB, EXEC_CB, ...) \\', + ' \\\n'.join(dispatch_calls), + '', + ] + config_path = os.path.join(state.backend.src_to_build, state.subdir, config_name) + mlog.log( + "Generating", config_name, 'into path', config_path, + "based on the specifed targets" + ) + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(config_path, "w", encoding='utf-8') as cout: + cout.write('\n'.join(config_file)) + + static_libs = [] + if baseline: + static_libs.append(self.gen_target( + state, config_name, sources, + baseline, prefix, True, kwargs + )) + + for test_result in dispatch_tests: + static_libs.append(self.gen_target( + state, config_name, sources, + test_result, prefix, + False, kwargs + )) + return [info, static_libs] + + def gen_target(self, state, config_name, sources, + test_result, prefix, + is_baseline, stlib_kwargs): + target_name = 'baseline' if is_baseline else test_result['target_name'] + args = [f'-D{prefix}HAVE_{df}' for df in test_result['defines']] + args += test_result['args'] + if is_baseline: + args.append(f'-D{prefix}MTARGETS_BASELINE') + else: + args.append(f'-D{prefix}MTARGETS_CURRENT={target_name}') + stlib_kwargs = stlib_kwargs.copy() + stlib_kwargs.update({ + 'sources': sources, + 'c_args': stlib_kwargs.get('c_args', []) + args, + 'cpp_args': stlib_kwargs.get('cpp_args', []) + args + }) + static_lib = state._interpreter.func_static_lib( + None, [config_name + '_' + target_name], + stlib_kwargs + ) + return static_lib @typed_pos_args('feature.sort', varargs=FeatureObject, min_varargs=1) - @noKwargs + @typed_kwargs('feature.sort', + KwargInfo('reverse', bool, default = False), + ) def sort_method(self, state: 'ModuleState', args: T.Tuple[T.List[FeatureObject]], kwargs: 'TYPE_kwargs' ) -> T.List[FeatureObject]: - return sorted(args[0]) + return sorted(args[0], reverse=kwargs['reverse']) - @staticmethod - def implicit(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: - implicit = set().union(*[fet.get_implicit() for fet in features]) - # since features can imply each other - implicit.difference_update(set(features)) - return sorted(implicit) + @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) + @noKwargs + def implicit_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + + features = args[0] + return sorted(self.implicit(features)) + + @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) + @noKwargs + def implicit_c_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + return sorted(self.implicit_c(args[0])) @staticmethod - def implicit_c(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: - implicit = set().union(*[fet.get_implicit() for fet in features], features) - return sorted(implicit) + def implicit(features: T.Sequence[FeatureObject]) -> T.Set[FeatureObject]: + implies = set().union(*[f.get_implicit() for f in features]) + return implies @staticmethod - def ahead(features: T.Sequence[FeatureObject]) -> T.List[FeatureObject]: - implicit = set().union(*[fet.get_implicit() for fet in features]) - ahead = [fet for fet in features if fet not in implicit] - if len(ahead) == 0: - # return the highest interested feature - # if all features imply each other - ahead = list(sorted(features, reverse=True)[:1]) - return ahead + def implicit_c(features: T.Sequence[FeatureObject]) -> T.Set[FeatureObject]: + return Module.implicit(features).union(features) diff --git a/mesonbuild/modules/feature/utils.py b/mesonbuild/modules/feature/utils.py index b51ad5fbbfd2..b681e96c169a 100644 --- a/mesonbuild/modules/feature/utils.py +++ b/mesonbuild/modules/feature/utils.py @@ -1,36 +1,7 @@ # Copyright (c) 2023, NumPy Developers. # All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the NumPy Developers nor the names of any -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import typing as T - from ...mesonlib import MesonException, MachineChoice if T.TYPE_CHECKING: @@ -53,12 +24,8 @@ def get_compiler(state: 'ModuleState') -> 'Compiler': return compiler def test_code(state: 'ModuleState', compiler: 'Compiler', - args: T.List[str], headers: T.List[str], - code: 'T.Union[str, File]' + args: T.List[str], code: 'T.Union[str, File]' ) -> T.Tuple[bool, bool, str]: - if isinstance(code, str): - heads = '\n'.join([f'#include <{h}>' for h in headers]) - code = heads + '\n' + code # TODO: treat warnings as errors with compiler.cached_compile( code, state.environment.coredata, extra_args=args diff --git a/mesonbuild/modules/feature/x86_features.py b/mesonbuild/modules/feature/x86_features.py deleted file mode 100644 index 5d943b202021..000000000000 --- a/mesonbuild/modules/feature/x86_features.py +++ /dev/null @@ -1,554 +0,0 @@ -# Copyright (c) 2023, NumPy Developers. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the NumPy Developers nor the names of any -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import typing as T -from textwrap import dedent - -from ... import mlog -from .feature import FeatureObject - -if T.TYPE_CHECKING: - from ...compilers import Compiler - from ...interpreterbase import TYPE_var, TYPE_kwargs - from .. import ModuleState - -def test_code_main(code: str) -> str: - return dedent(f'''\ - int main(int, char **argv) - {{ - char *src = argv[1]; - {dedent(code)} - return 0; - }} - ''') - -def test_code(NAME: str, code: str) -> str: - return dedent(f'''\ - #if defined(DETECT_FEATURES) && defined(__{NAME}__) - #error "HOST/ARCH doesn't support {NAME}" - #endif - {test_code_main(code)} - ''') - -def feature(state: 'ModuleState', name: str, interest: int, - kwargs: 'TYPE_kwargs') -> FeatureObject: - code = kwargs.get('test_code') - extra_tests = kwargs.get('extra_tests') - if isinstance(code, str): - kwargs['test_code'] = test_code(name, code) - if isinstance(extra_tests, dict): - kwargs['extra_tests'] = { - tname: test_code_main(code) if isinstance(code, str) else code - for tname, code in extra_tests.items() - } - return FeatureObject(state, [name, interest], kwargs) - -def _init_features(state: 'ModuleState') -> T.Dict[str, FeatureObject]: - features: T.Dict[str, FeatureObject] = {} - fet: T.Callable[[str, int, 'TYPE_kwargs'], FeatureObject] = \ - lambda name, interest, kwargs: features.setdefault( - name, feature(state, name, interest, kwargs) - ) - - cpu_family = state.build_machine.cpu_family - is_x86 = cpu_family in ('x86', 'x86_64', 'x64') - SSE = fet('SSE', 1, dict( - headers='xmmintrin.h', - disable=( - f'not supported by build machine "{cpu_family}"' - if not is_x86 else '' - ), - test_code='''\ - __m128 s0 = _mm_loadu_ps((float*)src); - __m128 s1 = _mm_loadu_ps((float*)src+4); - __m128 rt = _mm_add_ps(s0, s1); - _mm_storeu_ps((float*)src, rt); - ''' - )) - SSE2 = fet('SSE2', 2, dict( - implies=SSE, headers='emmintrin.h', - test_code='''\ - __m128i s0 = _mm_loadu_si128((__m128i*)src); - __m128i s1 = _mm_loadu_si128((__m128i*)src+16); - __m128i rt = _mm_add_epi16(s0, s1); - _mm_storeu_si128((__m128i*)src, rt); - ''' - )) - SSE3 = fet('SSE3', 3, dict( - implies=SSE2, headers='pmmintrin.h', - test_code='''\ - __m128 s0 = _mm_loadu_ps((float*)src); - __m128 s1 = _mm_loadu_ps((float*)src+4); - __m128 rt = _mm_hadd_ps(s0, s1); - _mm_storeu_ps((float*)src, rt); - ''' - )) - if state.build_machine.is_64_bit: - SSE.update_method(state, [], dict(implies=[SSE2, SSE3])) - SSE2.update_method(state, [], dict(implies=[SSE, SSE3])) - - SSSE3 = fet('SSSE3', 4, dict( - implies=SSE3, headers='tmmintrin.h', - test_code='''\ - __m128i s0 = _mm_loadu_si128((__m128i*)src); - __m128i s1 = _mm_loadu_si128((__m128i*)src+16); - __m128i rt = _mm_hadd_epi16(s0, s1); - _mm_storeu_si128((__m128i*)src, rt); - ''' - )) - SSE41 = fet('SSE41', 5, dict( - implies=SSSE3, headers='smmintrin.h', - test_code='''\ - __m128 s0 = _mm_loadu_ps((float*)src); - __m128 rt = _mm_ceil_ps(s0); - _mm_storeu_ps((float*)src, rt); - ''' - )) - POPCNT = fet('POPCNT', 6, dict( - implies=SSE41, headers='popcntintrin.h', - test_code='''\ - unsigned long long a = *((unsigned long long*)src); - unsigned int b = *((unsigned int*)src+4); - int rt; - #if defined(_M_X64) || defined(__x86_64__) - a = _mm_popcnt_u64(a); - #endif - b = _mm_popcnt_u32(b); - rt = (int)a + (int)b; - _mm_storeu_si128((__m128i*)src, _mm_set1_epi32(rt)); - ''' - )) - SSE42 = fet('SSE42', 7, dict( - implies=POPCNT, - test_code='''\ - __m128i s0 = _mm_loadu_si128((__m128i*)src); - __m128i s1 = _mm_loadu_si128((__m128i*)src+16); - __m128i rt = _mm_cmpgt_epi64(s0, s1); - _mm_storeu_si128((__m128i*)src, rt); - ''' - )) - # 7-20 left as margin for any extra features - AVX = fet('AVX', 20, dict( - implies=SSE42, - headers='immintrin.h', - detect=dict(val='AVX', match='.*SSE.*'), - test_code='''\ - __m256 s0 = _mm256_loadu_ps((float*)src); - __m256 s1 = _mm256_loadu_ps((float*)src+8); - __m256 rt = _mm256_add_ps(s0, s1); - _mm256_storeu_ps((float*)src, rt); - ''' - )) - # Amd abandon these two features, should we remove them? - XOP = fet('XOP', 21, dict( - implies=AVX, headers='x86intrin.h', - test_code='''\ - __m128i s0 = _mm_loadu_si128((__m128i*)src); - __m128i s1 = _mm_loadu_si128((__m128i*)src+16); - __m128i rt = _mm_comge_epu32(s0, s1); - _mm_storeu_si128((__m128i*)src, rt); - ''' - )) - FMA4 = fet('FMA4', 22, dict( - implies=AVX, headers='x86intrin.h', - test_code='''\ - __m128 s0 = _mm_loadu_ps((float*)src); - __m128 s1 = _mm_loadu_ps((float*)src+4); - __m128 s2 = _mm_loadu_ps((float*)src+8); - __m128 rt = _mm256_macc_ps(s0, s1, s2); - _mm_storeu_ps((float*)src, rt); - ''' - )) - # x86 half-precision - F16C = fet('F16C', 23, dict( - implies=AVX, - test_code=f'''\ - __m128i s0 = _mm_loadu_si128((__m128i*)src); - __m128 rt = _mm_cvtph_ps(s0); - _mm_storeu_ps((float*)src, rt); - ''' - )) - FMA3 = fet('FMA3', 24, dict( - implies=F16C, - test_code=f'''\ - __m256 s0 = _mm256_loadu_ps((float*)src); - __m256 s1 = _mm256_loadu_ps((float*)src + 8); - __m256 s2 = _mm256_loadu_ps((float*)src + 16); - __m256 rt = _mm256_fmadd_ps(s0, s1, s2); - _mm256_storeu_ps((float*)src, rt); - ''' - )) - AVX2 = fet('AVX2', 25, dict( - implies=F16C, - test_code=f'''\ - __m256i s0 = _mm256_loadu_si256((__m256i*)src); - __m256i rt = _mm256_abs_epi16(s0); - _mm256_storeu_si256((__m256i*)src, rt); - ''' - )) - # 25-40 left as margin for any extra features - AVX512_COMMON = fet('AVX512_COMMON', 40, dict( - implies=[FMA3, AVX2], - group = ['AVX512F', 'AVX512CD'], - detect = [ - dict(val='AVX512F', match='.*'), - 'AVX512CD' - ], - test_code=f'''\ - __m512i s0 = _mm512_loadu_si512((__m512i*)src); - __m512i rt = _mm512_abs_epi32(s0); - /* avx512cd */ - rt = _mm512_lzcnt_epi32(rt); - _mm512_storeu_si512((__m512i*)src, rt); - ''', - extra_tests = dict( - AVX512F_REDUCE=f'''\ - __m512i si = _mm512_loadu_si512((__m512i*)src); - __m512 ps = _mm512_loadu_ps((__m512*)src); - __m512d pd = _mm512_loadu_pd((__m512d*)src); - /* add */ - float sum_ps = _mm512_reduce_add_ps(ps); - double sum_pd = _mm512_reduce_add_pd(pd); - int sum_int = (int)_mm512_reduce_add_epi64(si); - sum_int += (int)_mm512_reduce_add_epi32(si); - /* mul */ - sum_ps += _mm512_reduce_mul_ps(ps); - sum_pd += _mm512_reduce_mul_pd(pd); - sum_int += (int)_mm512_reduce_mul_epi64(si); - sum_int += (int)_mm512_reduce_mul_epi32(si); - /* min */ - sum_ps += _mm512_reduce_min_ps(ps); - sum_pd += _mm512_reduce_min_pd(pd); - sum_int += (int)_mm512_reduce_min_epi32(si); - sum_int += (int)_mm512_reduce_min_epu32(si); - sum_int += (int)_mm512_reduce_min_epi64(si); - /* max */ - sum_ps += _mm512_reduce_max_ps(ps); - sum_pd += _mm512_reduce_max_pd(pd); - sum_int += (int)_mm512_reduce_max_epi32(si); - sum_int += (int)_mm512_reduce_max_epu32(si); - sum_int += (int)_mm512_reduce_max_epi64(si); - /* and */ - sum_int += (int)_mm512_reduce_and_epi32(si); - sum_int += (int)_mm512_reduce_and_epi64(si); - /* or */ - sum_int += (int)_mm512_reduce_or_epi32(si); - sum_int += (int)_mm512_reduce_or_epi64(si); - sum_int += (int)sum_ps + (int)sum_pd; - *(int*)src = sum_int; - ''' - ) - )) - AVX512_KNL = fet('AVX512_KNL', 41, dict( - implies=AVX512_COMMON, - group = ['AVX512ER', 'AVX512PF'], - test_code=f'''\ - int base[128]; - __m512d ad = _mm512_loadu_pd((__m512d*)src); - /* ER */ - __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); - /* PF */ - _mm512_mask_prefetch_i64scatter_pd( - base, _mm512_cmpeq_epi64_mask(a, a), a, 1, _MM_HINT_T1 - ); - *(int*)src = base[0]; - ''' - )) - AVX512_KNM = fet('AVX512_KNM', 42, dict( - implies=AVX512_KNL, - group = ['AVX5124FMAPS', 'AVX5124VNNIW', 'AVX512VPOPCNTDQ'], - test_code=f'''\ - __m512i si = _mm512_loadu_si512((__m512i*)src); - __m512 ps = _mm512_loadu_ps((__m512*)src + 64); - /* 4FMAPS */ - ps = _mm512_4fmadd_ps(ps, ps, ps, ps, ps, NULL); - /* 4VNNIW */ - si = _mm512_4dpwssd_epi32(si, si, si, si, si, NULL); - /* VPOPCNTDQ */ - si = _mm512_popcnt_epi64(si); - si = _mm512_add_epi32(si, _mm512_castps_si512(ps)); - _mm512_storeu_si512((__m512i*)src, si); - ''' - )) - AVX512_SKX = fet('AVX512_SKX', 43, dict( - implies=AVX512_COMMON, - group = ['AVX512VL', 'AVX512BW', 'AVX512DQ'], - test_code=f'''\ - __m512i aa = _mm512_abs_epi32(_mm512_loadu_si512((__m512i*)src)); - /* VL */ - __m256i a = _mm256_abs_epi64(_mm512_extracti64x4_epi64(aa, 1)); - /* DQ */ - __m512i b = _mm512_broadcast_i32x8(a); - /* BW */ - b = _mm512_abs_epi16(b); - _mm512_storeu_si512((__m512i*)src, b); - ''', - extra_tests = dict( - AVX512BW_MASK=f'''\ - __m512i s0 = _mm512_loadu_si512((__m512i*)src); - __m512i s1 = _mm512_loadu_si512((__m512i*)src + 64); - __mmask64 m64 = _mm512_cmpeq_epi8_mask(s0, s1); - m64 = _kor_mask64(m64, m64); - m64 = _kxor_mask64(m64, m64); - m64 = _cvtu64_mask64(_cvtmask64_u64(m64)); - m64 = _mm512_kunpackd(m64, m64); - m64 = (__mmask64)_mm512_kunpackw( - (__mmask32)m64, (__mmask32)m64 - ); - *(int*)src = (int)_cvtmask64_u64(m64); - ''', - AVX512DQ_MASK=f'''\ - __m512i s0 = _mm512_loadu_si512((__m512i*)src); - __m512i s1 = _mm512_loadu_si512((__m512i*)src + 64); - __mmask8 m8 = _mm512_cmpeq_epi64_mask(s0, s1); - m8 = _kor_mask8(m8, m8); - m8 = _kxor_mask8(m8, m8); - m8 = _cvtu32_mask8(_cvtmask8_u32(m8)); - *(int*)src = (int)_cvtmask8_u32(m8); - ''' - ) - )) - AVX512_CLX = fet('AVX512_CLX', 44, dict( - implies=AVX512_SKX, - group = 'AVX512VNNI', - test_code=f'''\ - /* VNNI */ - __m512i a = _mm512_loadu_si512((__m512i*)src); - a = _mm512_dpbusd_epi32(a, _mm512_setzero_si512(), a); - _mm512_storeu_si512((__m512i*)src, a); - ''' - )) - AVX512_CNL = fet('AVX512_CNL', 45, dict( - implies=AVX512_SKX, - group = ['AVX512IFMA', 'AVX512VBMI'], - test_code=f'''\ - __m512i a = _mm512_loadu_si512((const __m512i*)src); - /* IFMA */ - a = _mm512_madd52hi_epu64(a, a, _mm512_setzero_si512()); - /* VMBI */ - a = _mm512_permutex2var_epi8(a, _mm512_setzero_si512(), a); - _mm512_storeu_si512((__m512i*)src, a); - ''' - )) - AVX512_ICL = fet('AVX512_ICL', 46, dict( - implies=[AVX512_CLX, AVX512_CNL], - group = ['AVX512VBMI2', 'AVX512BITALG', 'AVX512VPOPCNTDQ'], - test_code=f'''\ - __m512i a = _mm512_loadu_si512((__m512i*)src); - /* VBMI2 */ - a = _mm512_shrdv_epi64(a, a, _mm512_setzero_si512()); - /* BITLAG */ - a = _mm512_popcnt_epi8(a); - /* VPOPCNTDQ */ - a = _mm512_popcnt_epi64(a); - _mm512_storeu_si512((__m512i*)src, a); - ''' - )) - return features - -def _features_args(state: 'ModuleState', - compiler: 'Compiler', - features: T.Dict[str, FeatureObject]) -> None: - cid = compiler.get_id() - # for both unix-like and msvc-like - if 'intel' in cid: - # Intel Compiler doesn't support AVX2 or FMA3 independently - for fet, implies in ( - 'FMA3', ('F16C', 'AVX2'), - 'AVX2', ('F16C', 'FMA3'), - ): - features[fet].update_method(state, [], { - 'implies': [features[sub_fet] for sub_fet in implies] - }) - for dis_fet in ('XOP', 'FMA4'): - features[dis_fet].update_method(state, [], { - 'disable': 'Intel Compiler does not support it' - }) - if 'intel-cl' in cid: - for fet in ( - 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'AVX' - ): - features[fet].update_method(state, [], { - 'args': dict(val='/arch:' + fet, match='/arch:.*') - }) - # POPCNT, and F16C don't own private FLAGS still - # Intel's compiler provides ABI capability for them. - for fet, arg in ( - ('SSE42', dict( - val='/arch:SSE4.2', match='/arch:.*' - )), - ('FMA3', dict( - val='/arch:CORE-AVX2', match='/arch:.*' - )), - ('AVX2', dict( - val='/arch:CORE-AVX2', match='/arch:.*' - )), - ('AVX512_COMMON', dict( - val='/Qx:COMMON-AVX512', match='/arch:.*' - )), - ('AVX512_KNL', dict( - val='/Qx:KNL', match='/[arch|Qx]:.*' - )), - ('AVX512_KNM', dict( - val='/Qx:KNM', match='/[arch|Qx]:.*' - )), - ('AVX512_SKX', dict( - val='/Qx:SKYLAKE-AVX512', match='/[arch|Qx]:.*' - )), - ('AVX512_CLX', dict( - val='/Qx:CASCADELAKE', match='/[arch|Qx]:.*' - )), - ('AVX512_CNL', dict( - val='/Qx:CANNONLAKE', match='/[arch|Qx]:.*' - )), - ('AVX512_ICL', dict( - val='/Qx:ICELAKE-CLIENT', match='/[arch|Qx]:.*' - )), - ): - features[fet].update_method(state, [], {'args': arg}) - elif 'intel' in cid: - for fet in ( - 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'AVX' - ): - features[fet].update_method(state, [], { - 'args': '-m' + fet.lower() - }) - # POPCNT, and F16C don't own private FLAGS still - # Intel's compiler provides ABI capability for them. - - # we specify the arguments that we need to filterd - # from implied features rather than blinedly removes all args(.*) - # since args may updated from outside. - filter_all = r'.*m[sse|avx|arch\=|-x[a-z0-9\-]].*' - for fet, arg in ( - ('SSE41', dict(val='-msse4.1')), - ('SSE42', dict(val='-msse4.2')), - ('FMA3', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), - ('AVX2', dict(val='-march=core-avx2', match='.*m[sse|avx].*')), - ('AVX512_COMMON', dict( - val='-march=common-avx512', match=r'.*m[sse|avx|arch\=].*' - )), - ('AVX512_KNL', dict( - val='-xKNL', match=filter_all - )), - ('AVX512_KNM', dict( - val='-xKNM', match=filter_all - )), - ('AVX512_SKX', dict( - val='-xSKYLAKE-AVX512', match=filter_all - )), - ('AVX512_CLX', dict( - val='-xCASCADELAKE', match=filter_all - )), - ('AVX512_CNL', dict( - val='-xCANNONLAKE', match=filter_all - )), - ('AVX512_ICL', dict( - val='-xICELAKE-CLIENT', match=filter_all - )), - ): - features[fet].update_method(state, [], {'args': arg}) - elif 'msvc' in cid: - # SSE3, SSSE3, SSE41, POPCNT, SSE42, F16C, XOP and FMA4 - # don't own private FLAGS still MS's compiler provides - # ABI capability for them. - for fet, arg in ( - ('SSE', dict(val='/arch:SSE')), - ('SSE2', dict(val='/arch:SSE2', match='/arch:.*')), - ('AVX', dict(val='/arch:AVX', match='/arch:.*')), - ('AVX2', dict(val='/arch:AVX2', match='/arch:.*')), - ('FMA3', dict(val='/arch:AVX2', match='/arch:.*')), - ('AVX512_COMMON', dict(val='/arch:AVX512', match='/arch:.*')), - ('AVX512_SKX', dict(val='/arch:AVX512', match='/arch:.*')), - ): - features[fet].update_method(state, [], {'args': arg}) - # MSVC special headers - for fet, header in ( - ('POPCNT', 'nmmintrin.h'), - ('XOP', 'ammintrin.h'), - ('FMA4', 'ammintrin.h'), - ): - features[fet].update_method(state, [], {'headers': header}) - - # MSVC doesn't support FMA3 or AVX2 independently - # same for AVX512_COMMON and AVX512_SKX - for fet, implies in ( - 'FMA3', ('F16C', 'AVX2'), - 'AVX2', ('F16C', 'FMA3'), - 'AVX512_COMMON', ('FMA3', 'AVX2', 'AVX512_SKX'), - ): - features[fet].update_method(state, [], { - 'implies': [features[sub_fet] for sub_fet in implies] - }) - for dis_fet in ('AVX512_KNL', 'AVX512_KNM'): - features[dis_fet].update_method(state, [], { - 'disable': 'MSVC compiler does not support it' - }) - # unix-like (gcc, clang) - else: - for fet in ( - 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'POPCNT', - 'AVX', 'F16C', 'XOP', 'FMA4', 'AVX2' - ): - features[fet].update_method(state, [], { - 'args': '-m' + fet.lower() - }) - - for fet, args in ( - ('SSE41', '-msse4.1'), - ('SSE42', '-msse4.2'), - ('FMA3', '-mfma'), - # should we add "-mno-mmx"? - ('AVX512_COMMON', '-mavx512f -mavx512cd'), - ('AVX512_KNL', '-mavx512er -mavx512pf'), - ('AVX512_KNM', '-mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq'), - ('AVX512_SKX', '-mavx512vl -mavx512bw -mavx512dq'), - ('AVX512_CLX', '-mavx512vnni'), - ('AVX512_CNL', '-mavx512ifma -mavx512vbmi'), - ('AVX512_ICL', '-mavx512vbmi2 -mavx512bitalg -mavx512vpopcntdq') - ): - features[fet].update_method(state, [], { - 'args': args.split() - }) - - for fet in ( - 'SSE', 'SSE2', 'SSE3', 'SSSE3', 'POPCNT', - 'AVX', 'F16C', 'XOP', 'FMA4', 'AVX2' - ): - features[fet].update_method(state, [], { - 'args': '-m' + fet.lower() - }) - -def x86_features(state: 'ModuleState', compiler: 'Compiler' - ) -> T.Dict[str, FeatureObject]: - features: T.Dict[str, FeatureObject] = _init_features(state) - _features_args(state, compiler, features) - return features From 7d8bbe1dceaebb4d6a29385a6f6eaa5ce5b056cf Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 19 Jul 2023 04:05:52 +0400 Subject: [PATCH 09/25] Fix config src dir --- mesonbuild/modules/feature/module.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py index ad56a1d59dc2..7b2e4a138b17 100644 --- a/mesonbuild/modules/feature/module.py +++ b/mesonbuild/modules/feature/module.py @@ -390,7 +390,13 @@ def multi_target_method(self, state: 'ModuleState', ' \\\n'.join(dispatch_calls), '', ] - config_path = os.path.join(state.backend.src_to_build, state.subdir, config_name) + + src_dir = state.environment.source_dir + sub_dir = state.subdir + if sub_dir: + src_dir = os.path.join(src_dir, state.subdir) + config_path = os.path.abspath(os.path.join(src_dir, config_name)) + mlog.log( "Generating", config_name, 'into path', config_path, "based on the specifed targets" From 6cb3f994418ef2926330cab2dd1e3b6b48268127 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 7 Aug 2023 14:25:48 +0400 Subject: [PATCH 10/25] Several improvements - Improve multi_targets() method: * rename multi_target() to multi_targets() use plural to indicates handling more than one CPU target * Split the function into multiple smaller functions for better organization. * Modify the function to return a single library array instead of a list of libraries. * Avoid returning the result of testing the enabled features to keep it * simple. * Enhance debugging capabilities by printing the test results of enabled targets. * Avoid returns static_lib instead wraps it with another object to reduce the number of final generated objects also to sort the all objects based on lowest interest * validate the sort of dispatch features - fix mypy - init the module doc - allow disables the cache - fix method `features.new` name during trigger arguments errors - generate config files inside the builddir instead - add test cases - cache tests within coredata - fix duplicate baseline build - refactor module name from `feature` to `features` - fix python typying --- docs/markdown/Features-module.md | 259 +++++++ docs/sitemap.txt | 1 + mesonbuild/modules/feature/__init__.py | 41 - mesonbuild/modules/feature/module.py | 481 ------------ mesonbuild/modules/features/__init__.py | 13 + .../modules/{feature => features}/feature.py | 139 ++-- mesonbuild/modules/features/module.py | 714 ++++++++++++++++++ .../modules/{feature => features}/utils.py | 7 + run_unittests.py | 0 test cases/features/1 baseline/baseline.c | 77 ++ test cases/features/1 baseline/init_features | 1 + test cases/features/1 baseline/meson.build | 14 + .../features/2 multi_targets/dispatch.h | 90 +++ .../features/2 multi_targets/dispatch1.c | 28 + .../features/2 multi_targets/dispatch2.c | 21 + .../features/2 multi_targets/dispatch3.c | 1 + .../features/2 multi_targets/init_features | 1 + test cases/features/2 multi_targets/main.c | 45 ++ .../features/2 multi_targets/meson.build | 26 + .../features/init_features/checks/cpu_asimd.c | 27 + .../features/init_features/checks/cpu_neon.c | 19 + .../init_features/checks/cpu_neon_fp16.c | 11 + .../init_features/checks/cpu_neon_vfpv4.c | 21 + .../features/init_features/checks/cpu_sse.c | 7 + .../features/init_features/checks/cpu_sse2.c | 7 + .../features/init_features/checks/cpu_sse3.c | 7 + .../features/init_features/checks/cpu_sse41.c | 7 + .../features/init_features/checks/cpu_ssse3.c | 7 + test cases/features/init_features/meson.build | 98 +++ unittests/featurestests.py | 300 ++++++++ 30 files changed, 1890 insertions(+), 580 deletions(-) create mode 100644 docs/markdown/Features-module.md delete mode 100644 mesonbuild/modules/feature/__init__.py delete mode 100644 mesonbuild/modules/feature/module.py create mode 100644 mesonbuild/modules/features/__init__.py rename mesonbuild/modules/{feature => features}/feature.py (74%) create mode 100644 mesonbuild/modules/features/module.py rename mesonbuild/modules/{feature => features}/utils.py (83%) mode change 100755 => 100644 run_unittests.py create mode 100644 test cases/features/1 baseline/baseline.c create mode 120000 test cases/features/1 baseline/init_features create mode 100644 test cases/features/1 baseline/meson.build create mode 100644 test cases/features/2 multi_targets/dispatch.h create mode 100644 test cases/features/2 multi_targets/dispatch1.c create mode 100644 test cases/features/2 multi_targets/dispatch2.c create mode 100644 test cases/features/2 multi_targets/dispatch3.c create mode 120000 test cases/features/2 multi_targets/init_features create mode 100644 test cases/features/2 multi_targets/main.c create mode 100644 test cases/features/2 multi_targets/meson.build create mode 100644 test cases/features/init_features/checks/cpu_asimd.c create mode 100644 test cases/features/init_features/checks/cpu_neon.c create mode 100644 test cases/features/init_features/checks/cpu_neon_fp16.c create mode 100644 test cases/features/init_features/checks/cpu_neon_vfpv4.c create mode 100644 test cases/features/init_features/checks/cpu_sse.c create mode 100644 test cases/features/init_features/checks/cpu_sse2.c create mode 100644 test cases/features/init_features/checks/cpu_sse3.c create mode 100644 test cases/features/init_features/checks/cpu_sse41.c create mode 100644 test cases/features/init_features/checks/cpu_ssse3.c create mode 100644 test cases/features/init_features/meson.build create mode 100644 unittests/featurestests.py diff --git a/docs/markdown/Features-module.md b/docs/markdown/Features-module.md new file mode 100644 index 000000000000..a0b07ad4b577 --- /dev/null +++ b/docs/markdown/Features-module.md @@ -0,0 +1,259 @@ +# Features module + +## Overview + +Dealing with numerous CPU features through C and C++ compilers is a challenging task, +especially when aiming to support massive amount of CPU features for various architectures and multiple compilers +Additionally, supporting both baseline features and additional features dispatched at runtime presents another dilemma. + +Another issue that may arise is simplifying the implementations of generic interfaces while keeping the dirty work laid +on the build system rather than using nested namespaces or recursive sources, relying on pragma or compiler targets attributes +on top of complicated precompiled macros or meta templates, which can make debugging and maintenance difficult. + +While this module doesn't force you to follow a specific approach, it instead paves the way to count on a +practical multi-targets solution that can make managing CPU features easier and more reliable. + +In a nutshell, this module helps you deliver the following concept: + +```C +// Brings the headers files of enabled CPU features +#ifdef HAVE_SSE + #include +#endif +#ifdef HAVE_SSE2 + #include +#endif +#ifdef HAVE_SSE3 + #include +#endif +#ifdef HAVE_SSSE3 + #include +#endif +#ifdef HAVE_SSE41 + #include +#endif +#ifdef HAVE_POPCNT + #ifdef _MSC_VER + #include + #else + #include + #endif +#endif +#ifdef HAVE_AVX + #include +#endif + +#ifdef HAVE_NEON + #include +#endif + +// MTARGETS_CURRENT defined as compiler argument via `features.multi_targets()` +#ifdef MTARGETS_CURRENT + #define TARGET_SFX(X) X##_##MTARGETS_CURRENT +#else + // baseline or when building source without this module. + #define TARGET_SFX(X) X +#endif + +void TARGET_SFX(my_kernal)(const float *src, float *dst) +{ +#ifdef HAVE_AVX512F + // defeintions for implied features alawys present + // no matter the compiler is + #ifndef HAVE_AVX2 + #error "Alawys defined" + #endif +#elif defined(HAVE_AVX2) && defined(HAVE_FMA3) + #ifndef HAVE_AVX + #error "Alawys defined" + #endif +#elif defined(HAVE_SSE41) + #ifndef HAVE_SSSE3 + #error "Alawys defined" + #endif +#elif defined(HAVE_SSE2) + #ifndef HAVE_SSE2 + #error "Alawys defined" + #endif +#elif defined(HAVE_ASIMDHP) + #if !defined(HAVE_NEON) || !defined(HAVE_ASIMD) + #error "Alawys defined" + #endif +#elif defined(HAVE_ASIMD) + #ifndef HAVE_NEON_VFPV4 + #error "Alawys defined" + #endif +#elif defined(HAVE_NEON_F16) + #ifndef HAVE_NEON + #error "Alawys defined" + #endif +#else + // fallback to C scalar +#endif +} +``` + +From the above code we can deduce the following: +- The code is written on top features based definitions rather than counting clusters or + features groups which gives the code more readability and flexibility. + +- Avoid using compiler built-in defeintions no matters the enabled arguments allows you + to easily manage the enabled/disabled features and to deal with any kind of compiler or features. + Since compilers like MSVC for example doesn't provides defeintions for all CPU features. + +- The code is not aware of how its going to be build it, that gives the code a great prodiblity to + manage the generated objects which allow raising the baseline features at any time + or reduce and increase the additional dispatched features without changing the code. + +- Allow building a single source multiple of times simplifying the implementations + of generic interfaces. + + +## Usage + +To use this module, just do: **`features = import('features')`**. The +following functions will then be available as methods on the object +with the name `features`. You can, of course, replace the name `features` +with anything else. + +### features.new() +```meson +features.new(string, int, + implies: FeatureOject | FeatureObject[] = [], + group: string | string[] = [], + detect: string | {} | (string | {})[] = [], + args: string | {} | (string | {})[] = [], + test_code: string | File = '', + extra_tests: {string: string | file} = {}, + disable: string = '' + ) -> FeatureObject +``` + +This function plays a crucial role in the Features Module as it creates +a new `FeatureObject` instance that is essential for the functioning of +other methods within the module. + +It takes two required positional arguments. The first one is the name of the feature, +and the second is the interest level of the feature, which is used by +the sort operation and priority of conflicting arguments. +The rest of the kwargs arguments are explained as follows: + +* `implies` **FeatureOject | FeatureObject[] = []**: One or an array of features objects + representing predecessor features. + +* `group` **string | string[] = []**: Optional one or an array of extra features names + to be added as extra definitions that can passed to source. + +* `args` **string | {} | (string | {})[] = []**: Optional one or an array of compiler + arguments that are required to be enabled for this feature. + Each argument can be a string or a dictionary that holds four items that allow dealing + with the conflicts of the arguments of implied features: + - `val` **string**: string, the compiler argument. + - `match` **string | empty**: regex to match the arguments of implied features + that need to be filtered or erased. + - `mfilter` **string | empty**: regex to find certain strings from the matched arguments + to be combined with `val`. If the value of `mfilter` is empty or undefined, + any matches triggered by the value of `match` will not be combined with `val`. + - `mjoin` **string | empty**: a separator used to join all the filtered arguments. + If it's empty or undefined, the filtered arguments will be joined without a separator. + +* `detect` **string | {} | (string | {})[] = []**: Optional one or an array of features names + that required to be detect on runtime. If no features sepecfied then the values of `group` + will be used if its provides otherwise the name of the feature will be used instead. + Similar to `args`, each feature name can be a string or a dictionary that holds four items + that allow dealing with the conflicts of the of implied features names. + See `features.multi_targets()` or `features.test()` for more clearfications. + +* `test_code` **string | File = ''**: Optional C/C++ code or the path to the source + that needs to be tested against the compiler to consider this feature is supported. + +* `extra_tests` **{string: string | file} = {}**: Optional dictionary holds extra tests where + the key represents the test name, which is also added as a compiler definition if the test succeeded, + and the value is C/C++ code or the path to the source that needs to be tested against the compiler. + +* `disable` **string = ''**: Optional string to consider this feature disabled. + +Returns a new instance of `FeatureObject`. + +Example: + +```Meson +cpu = host_machine.cpu_family() +features = import('features') + +ASIMD = features.new( + 'ASIMD', 1, group: ['NEON', 'NEON_VFPV4', 'NEON_VFPV4'], + args: cpu == 'aarch64' ? '' : [ + '-mfpu=neon-fp-armv8', + '-march=armv8-a+simd' + ], + disable: cpu in ['arm', 'aarch64'] ? '' : 'Not supported by ' + cpu +) +# ARMv8.2 half-precision & vector arithm +ASIMDHP = features.new( + 'ASIMDHP', 2, implies: ASIMD, + args: { + 'val': '-march=armv8.2-a+fp16', + # search for any argument starts with `-match=` + 'match': '-march=', + # gets any string starts with `+` and apended to the value of `val` + 'mfilter': '\+.*' + } +) +## ARMv8.2 dot product +ASIMDDP = features.new( + 'ASIMDDP', 3, implies: ASIMD, + args: {'val': '-march=armv8.2-a+dotprod', 'match': '-march=.*', 'mfilter': '\+.*'} +) +## ARMv8.2 Single & half-precision Multiply +ASIMDFHM = features.new( + 'ASIMDFHM', 4, implies: ASIMDHP, + args: {'val': '-march=armv8.2-a+fp16fml', 'match': '-march=.*', 'mfilter': '\+.*'} +) +``` + +### features.test() +```meson +features.test(FeatureObject..., + anyfet: bool = false, + force_args: string | string[] | empty = empty, + compiler: Compiler | empty = empty, + cached: bool = true, + ) -> {} +``` + +Test a one or set of features against the compiler and returns a dictionary +contains all required information that needed for building a source that +requires these features. + +### features.multi_targets() +```meson +features.multi_targets(string, ( + str | File | CustomTarget | CustomTargetIndex | + GeneratedList | StructuredSources | ExtractedObjects | + BuildTarget + )..., + dispatch: (FeatureObject | FeatureObject[])[] = [], + baseline: empty | FeatureObject[] = empty, + prefix: string = '', + compiler: empty | compiler = empty, + cached: bool = True + ) [{}[], StaticLibrary[]] +``` + + +### features.sort() +```meson +features.sort(FeatureObject..., reverse: bool = false) : FeatureObject[] +``` + +### features.implicit() +```meson +features.implicit(FeatureObject...) : FeatureObject[] +``` + +### features.implicit_c() +```meson +features.implicit_c(FeatureObject...) : FeatureObject[] +``` + diff --git a/docs/sitemap.txt b/docs/sitemap.txt index 6f0672b76644..8f313cf8bf69 100644 --- a/docs/sitemap.txt +++ b/docs/sitemap.txt @@ -61,6 +61,7 @@ index.md Windows-module.md i18n-module.md Wayland-module.md + Features-module.md Java.md Vala.md D.md diff --git a/mesonbuild/modules/feature/__init__.py b/mesonbuild/modules/feature/__init__.py deleted file mode 100644 index 6433d98eb442..000000000000 --- a/mesonbuild/modules/feature/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2023, NumPy Developers. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# -# * Neither the name of the NumPy Developers nor the names of any -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import typing as T - -from .module import Module - -if T.TYPE_CHECKING: - from ...interpreter import Interpreter - -def initialize(interpreter: 'Interpreter') -> Module: - return Module() - diff --git a/mesonbuild/modules/feature/module.py b/mesonbuild/modules/feature/module.py deleted file mode 100644 index 7b2e4a138b17..000000000000 --- a/mesonbuild/modules/feature/module.py +++ /dev/null @@ -1,481 +0,0 @@ -# Copyright (c) 2023, NumPy Developers. -# All rights reserved. - -import typing as T -import os - -from ... import mlog, build -from ...compilers import Compiler -from ...mesonlib import File, MesonException -from ...interpreter.type_checking import NoneType -from ...interpreterbase.decorators import ( - noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, - ContainerTypeInfo, permittedKwargs -) -from .. import ModuleInfo, NewExtensionModule, ModuleReturnValue -from .feature import FeatureObject, ConflictAttr -from .utils import test_code, get_compiler - -if T.TYPE_CHECKING: - from typing import TypedDict - from ...interpreterbase import TYPE_var, TYPE_kwargs - from .. import ModuleState - from .feature import FeatureKwArgs - - class TestKwArgs(TypedDict): - compiler: T.Optional[Compiler] - force_args: T.Optional[T.List[str]] - any: T.Optional[bool] - -class Module(NewExtensionModule): - INFO = ModuleInfo('feature', '0.1.0') - - def __init__(self) -> None: - super().__init__() - self.methods.update({ - 'new': self.new_method, - 'test': self.test_method, - 'implicit': self.implicit_method, - 'implicit_c': self.implicit_c_method, - 'sort': self.sort_method, - 'multi_target': self.multi_target_method, - }) - # TODO: How to store and load from files in meson? - self.cached_tests = {} - - def new_method(self, state: 'ModuleState', - args: T.List['TYPE_var'], - kwargs: 'TYPE_kwargs') -> FeatureObject: - return FeatureObject(state, args, kwargs) - - @typed_pos_args('feature.test', varargs=FeatureObject, min_varargs=1) - @typed_kwargs('feature.test', - KwargInfo('compiler', (NoneType, Compiler)), - KwargInfo('anyfet', bool, default = False), - KwargInfo( - 'force_args', (NoneType, str, ContainerTypeInfo(list, str)), - listify=True - ), - ) - def test_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TestKwArgs' - ) -> T.List[T.Union[bool, T.Dict[str, T.Any]]]: - - features = args[0] - features_set = set(features) - anyfet = kwargs['anyfet'] - compiler = kwargs.get('compiler') - if not compiler: - compiler = get_compiler(state) - - force_args = kwargs['force_args'] - if force_args is not None: - # removes in empty strings - force_args = [a for a in force_args if a] - - cached, test_result = self.test( - state, features=features_set, - compiler=compiler, - anyfet=anyfet, - force_args=force_args - ) - if not test_result['is_supported']: - if test_result['is_disabled']: - label = mlog.yellow('disabled') - else: - label = mlog.yellow('Unsupported') - else: - label = mlog.green('Supported') - if anyfet: - unsupported = [ - fet.name for fet in sorted(features_set) - if fet.name not in test_result['features'] - ] - if unsupported: - unsupported = ' '.join(unsupported) - label = mlog.green(f'Parial support, missing({unsupported})') - - features_names = ' '.join([f.name for f in features]) - log_prefix = f'Test features "{mlog.bold(features_names)}" :' - cached_msg = f'({mlog.blue("cached")})' if cached else '' - if not test_result['is_supported']: - mlog.log(log_prefix, label, 'due to', test_result['fail_reason']) - else: - mlog.log(log_prefix, label, cached_msg) - return [test_result['is_supported'], test_result] - - def test(self, state, features: T.Set[FeatureObject], - compiler: 'Compiler', - anyfet: bool = False, - force_args: T.Optional[T.Tuple[str]] = None, - _caller: T.Set[FeatureObject] = set() - ) -> T.Tuple[bool, T.Dict[str, T.Union[str, T.List[str]]]]: - # cached hash should inveolov all implied features - # since FeatureObject is mutable object. - implied_features = self.implicit(features) - test_hash = hash(( - tuple(sorted(features)), - tuple(sorted(implied_features)), - compiler, anyfet, - (-1 if force_args is None else tuple(force_args)) - )) - result = self.cached_tests.get(test_hash) - if result is not None: - return True, result - - all_features = sorted(implied_features.union(features)) - if anyfet: - cached, test_result = self.test( - state, features=features, - compiler=compiler, - force_args=force_args - ) - if test_result['is_supported']: - self.cached_tests[test_hash] = test_result - return False, test_result - - features_any = set() - for fet in all_features: - _, test_result = self.test( - state, features={fet,}, - compiler=compiler, - force_args=force_args - ) - if test_result['is_supported']: - features_any.add(fet) - - _, test_result = self.test( - state, features=features_any, - compiler=compiler, - force_args=force_args - ) - self.cached_tests[test_hash] = test_result - return False, test_result - - # For multiple features, it important to erase any features - # implied by another to avoid duplicate testing since - # implied already tested also we use this set to genrate - # unque target name that can be used for multiple targets - # build. - prevalent_features = features.difference(implied_features) - if len(prevalent_features) == 0: - # It happens when all features imply each other. - # Set the highest interested feature - prevalent_features = sorted(features)[-1:] - else: - prevalent_features = sorted(prevalent_features) - - prevalent_names = [fet.name for fet in prevalent_features] - # prepare the result dict - test_result = { - 'target_name': '__'.join(prevalent_names), - 'prevalent_features': prevalent_names, - 'features': [fet.name for fet in all_features], - 'args': [], - 'detect': [], - 'defines': [], - 'undefines': [], - 'is_supported': True, - 'is_disabled': False, - 'fail_reason': '', - } - def fail_result(fail_reason, is_disabled = False, - result_dict = test_result.copy()): - result_dict.update({ - 'is_supported': False, - 'is_disabled': is_disabled, - 'fail_reason': fail_reason, - 'features': [] - }) - self.cached_tests[test_hash] = result_dict - return False, result_dict - - # since we allows features to imply each other - # items of `features` may part of `implied_features` - _caller = _caller.union(prevalent_features) - predecessor_features = implied_features.difference(_caller) - for fet in sorted(predecessor_features): - _, pred_result = self.test( - state, features={fet,}, - compiler=compiler, - force_args=force_args, - _caller=_caller - ) - if not pred_result['is_supported']: - reason = f'Implied feature "{fet.name}" ' - pred_disabled = pred_result['is_disabled'] - if pred_disabled: - fail_reason = reason + 'is disabled' - else: - fail_reason = reason + 'is not supported' - return fail_result(fail_reason, pred_disabled) - - for k in ['defines', 'undefines']: - values = test_result[k] - pred_values = pred_result[k] - values += [v for v in pred_values if v not in values] - - # Sort based on the lowest interest to deal with conflict attributes - # when combine all attributes togathers - conflict_attrs = ['detect'] - if force_args is None: - conflict_attrs += ['args'] - else: - test_result['args'] = force_args - - for fet in all_features: - for attr in conflict_attrs: - values: T.List[ConflictAttr] = getattr(fet, attr) - accumulate_values = test_result[attr] - for conflict in values: - conflict_val: str = conflict.val - if not conflict.match: - accumulate_values.append(conflict_val) - continue - # select the acc items based on the match - new_acc: T.List[str] = [] - for acc in accumulate_values: - # not affected by the match so we keep it - if not conflict.match.match(acc): - new_acc.append(acc) - continue - # no filter so we totaly escape it - if not conflict.mfilter: - continue - filter_val = conflict.mfilter.findall(acc) - # no filter match so we totaly escape it - if not filter_val: - continue - conflict_val += conflict.mjoin.join(filter_val) - new_acc.append(conflict_val) - test_result[attr] = new_acc - - test_args = compiler.has_multi_arguments - args = test_result['args'] - if args: - supported_args, test_cached = test_args(args, state.environment) - if not supported_args: - return fail_result( - f'Arguments "{", ".join(args)}" are not supported' - ) - - for fet in prevalent_features: - if fet.disable: - return fail_result( - f'{fet.name} is disabled due to "{fet.disable}"', - fet.disable - ) - - if fet.test_code: - _, tested_code, _ = test_code( - state, compiler, args, fet.test_code - ) - if not tested_code: - return fail_result( - f'Compiler fails against the test code of "{fet.name}"' - ) - - test_result['defines'] += [fet.name] + fet.group - for extra_name, extra_test in fet.extra_tests.items(): - _, tested_code, _ = test_code( - state, compiler, args, extra_test - ) - k = 'defines' if tested_code else 'undefines' - test_result[k].append(extra_name) - - self.cached_tests[test_hash] = test_result - return False, test_result - - @permittedKwargs(build.known_stlib_kwargs | { - 'dispatch', 'baseline', 'prefix' - }) - @typed_pos_args('feature.multi_target', str, varargs=( - str, File, build.CustomTarget, build.CustomTargetIndex, - build.GeneratedList, build.StructuredSources, build.ExtractedObjects, - build.BuildTarget - )) - @typed_kwargs('feature.multi_target', - KwargInfo( - 'dispatch', ContainerTypeInfo(list, - (FeatureObject, list) - ), - default=[] - ), - KwargInfo( - 'baseline', (NoneType, ContainerTypeInfo(list, FeatureObject)) - ), - KwargInfo('prefix', str, default=''), - KwargInfo('compiler', (NoneType, Compiler)), - allow_unknown=True - ) - def multi_target_method(self, state: 'ModuleState', - args: T.Tuple[str], kwargs: 'TYPE_kwargs' - ) -> T.List[T.Union[T.Dict[str, str], T.Any]]: - config_name = args[0] - sources = args[1] - dispatch = kwargs.pop('dispatch') - baseline = kwargs.pop('baseline') - prefix = kwargs.pop('prefix') - compiler = kwargs.pop('compiler') - if not compiler: - compiler = get_compiler(state) - - info = {} - if baseline is not None: - baseline_features = self.implicit_c(baseline) - cached, baseline = self.test( - state, features=set(baseline), anyfet=True, - compiler=compiler - ) - info['BASELINE'] = baseline - else: - baseline_features = [] - - dispatch_tests = [] - for d in dispatch: - if isinstance(d, FeatureObject): - target = {d,} - is_base_part = d in baseline_features - else: - target = set(d) - is_base_part = all([f in baseline_features for f in d]) - if is_base_part: - # TODO: add log - continue - cached, test_result = self.test( - state, features=target, - compiler=compiler - ) - if not test_result['is_supported']: - continue - target_name = test_result['target_name'] - if target_name in info: - continue - info[target_name] = test_result - dispatch_tests.append(test_result) - - dispatch_calls = [] - for test_result in dispatch_tests: - detect = '&&'.join([ - f'TEST_CB({d})' for d in test_result['detect'] - ]) - if detect: - detect = f'({detect})' - else: - detect = '1' - target_name = test_result['target_name'] - dispatch_calls.append( - f'{prefix}_MTARGETS_EXPAND(' - f'EXEC_CB({detect}, {target_name}, __VA_ARGS__)' - ')' - ) - - config_file = [ - '/* Autogenerated by the Meson features module. */', - '/* Do not edit, your changes will be lost. */', - '', - f'#undef {prefix}_MTARGETS_EXPAND', - f'#define {prefix}_MTARGETS_EXPAND(X) X', - '', - f'#undef {prefix}MTARGETS_CONF_BASELINE', - f'#define {prefix}MTARGETS_CONF_BASELINE(EXEC_CB, ...) ' + ( - f'{prefix}_MTARGETS_EXPAND(EXEC_CB(__VA_ARGS__))' - if baseline is not None - else '' - ), - '', - f'#undef {prefix}MTARGETS_CONF_DISPATCH', - f'#define {prefix}MTARGETS_CONF_DISPATCH(TEST_CB, EXEC_CB, ...) \\', - ' \\\n'.join(dispatch_calls), - '', - ] - - src_dir = state.environment.source_dir - sub_dir = state.subdir - if sub_dir: - src_dir = os.path.join(src_dir, state.subdir) - config_path = os.path.abspath(os.path.join(src_dir, config_name)) - - mlog.log( - "Generating", config_name, 'into path', config_path, - "based on the specifed targets" - ) - os.makedirs(os.path.dirname(config_path), exist_ok=True) - with open(config_path, "w", encoding='utf-8') as cout: - cout.write('\n'.join(config_file)) - - static_libs = [] - if baseline: - static_libs.append(self.gen_target( - state, config_name, sources, - baseline, prefix, True, kwargs - )) - - for test_result in dispatch_tests: - static_libs.append(self.gen_target( - state, config_name, sources, - test_result, prefix, - False, kwargs - )) - return [info, static_libs] - - def gen_target(self, state, config_name, sources, - test_result, prefix, - is_baseline, stlib_kwargs): - target_name = 'baseline' if is_baseline else test_result['target_name'] - args = [f'-D{prefix}HAVE_{df}' for df in test_result['defines']] - args += test_result['args'] - if is_baseline: - args.append(f'-D{prefix}MTARGETS_BASELINE') - else: - args.append(f'-D{prefix}MTARGETS_CURRENT={target_name}') - stlib_kwargs = stlib_kwargs.copy() - stlib_kwargs.update({ - 'sources': sources, - 'c_args': stlib_kwargs.get('c_args', []) + args, - 'cpp_args': stlib_kwargs.get('cpp_args', []) + args - }) - static_lib = state._interpreter.func_static_lib( - None, [config_name + '_' + target_name], - stlib_kwargs - ) - return static_lib - - @typed_pos_args('feature.sort', varargs=FeatureObject, min_varargs=1) - @typed_kwargs('feature.sort', - KwargInfo('reverse', bool, default = False), - ) - def sort_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: - return sorted(args[0], reverse=kwargs['reverse']) - - @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) - @noKwargs - def implicit_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: - - features = args[0] - return sorted(self.implicit(features)) - - @typed_pos_args('feature.implicit', varargs=FeatureObject, min_varargs=1) - @noKwargs - def implicit_c_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: - return sorted(self.implicit_c(args[0])) - - @staticmethod - def implicit(features: T.Sequence[FeatureObject]) -> T.Set[FeatureObject]: - implies = set().union(*[f.get_implicit() for f in features]) - return implies - - @staticmethod - def implicit_c(features: T.Sequence[FeatureObject]) -> T.Set[FeatureObject]: - return Module.implicit(features).union(features) - diff --git a/mesonbuild/modules/features/__init__.py b/mesonbuild/modules/features/__init__.py new file mode 100644 index 000000000000..3b56affcfbe3 --- /dev/null +++ b/mesonbuild/modules/features/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. +# + +import typing as T + +from .module import Module + +if T.TYPE_CHECKING: + from ...interpreter import Interpreter + +def initialize(interpreter: 'Interpreter') -> Module: + return Module() diff --git a/mesonbuild/modules/feature/feature.py b/mesonbuild/modules/features/feature.py similarity index 74% rename from mesonbuild/modules/feature/feature.py rename to mesonbuild/modules/features/feature.py index e269d6231270..f4021484e28e 100644 --- a/mesonbuild/modules/feature/feature.py +++ b/mesonbuild/modules/features/feature.py @@ -4,16 +4,14 @@ import typing as T import re from dataclasses import dataclass, field -from enum import IntFlag, auto from ...mesonlib import File, MesonException from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( noKwargs, noPosargs, KwargInfo, typed_kwargs, typed_pos_args, - ContainerTypeInfo, noArgsFlattening + ContainerTypeInfo ) from .. import ModuleObject -from .utils import test_code, get_compiler if T.TYPE_CHECKING: from typing import TypedDict @@ -50,9 +48,6 @@ class ConflictAttr: ) mjoin: str = field(default='', hash=False, compare=False) - def __str__(self) -> str: - return self.val - def copy(self) -> 'ConflictAttr': return ConflictAttr(**self.__dict__) @@ -71,14 +66,12 @@ def to_dict(self) -> T.Dict[str, str]: class KwargConfilctAttr(KwargInfo): def __init__(self, func_name: str, opt_name: str, default: T.Any = None): - types = [ - str, ContainerTypeInfo(dict, str), + types = ( + NoneType, str, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, (dict, str)) - ] - if default is None: - types += [NoneType] + ) super().__init__( - opt_name, tuple(types), + opt_name, types, convertor = lambda values: self.convert( func_name, opt_name, values ), @@ -152,29 +145,6 @@ class FeatureUpdateKwArgs(FeatureKwArgs): interest: NotRequired[int] class FeatureObject(ModuleObject): - """ - A data class that represents the feature. - - A feature is a unit of work that can be developed, tested, and deployed independently. - - Attributes: - name: The name of the feature. - interest: The interest level of the feature. - It used for sorting and to determine succor features. - implies: A set of features objects that are implied by this feature. - (Optional) - This means that if this feature is enabled, - then the implied features will also be enabled. - If any of the implied features is not supported by the platform - or the compiler this feature will also considerd not supported. - group: A list of - - detect: A list of strings that identify the methods that can be used to detect whether the feature is supported. - args: A list of strings that identify the arguments that are required to enable this feature. - test_code: A string or a file object that contains the test code for this feature. - extra_tests: A dictionary that maps from the name of a test to the test code for that test. - disable: A string that specifies why this feature is disabled. - """ name: str interest: int implies: T.Set['FeatureObject'] @@ -191,8 +161,8 @@ def __init__(self, state: 'ModuleState', super().__init__() - @typed_pos_args('feature.new', str, int) - @typed_kwargs('feature.new', + @typed_pos_args('features.new', str, int) + @typed_kwargs('features.new', KwargInfo( 'implies', (FeatureObject, ContainerTypeInfo(list, FeatureObject)), @@ -202,8 +172,8 @@ def __init__(self, state: 'ModuleState', 'group', (str, ContainerTypeInfo(list, str)), default=[], listify=True ), - KwargConfilctAttr('feature.new', 'detect', default=[]), - KwargConfilctAttr('feature.new', 'args', default=[]), + KwargConfilctAttr('features.new', 'detect', default=[]), + KwargConfilctAttr('features.new', 'args', default=[]), KwargInfo('test_code', (str, File), default=''), KwargInfo( 'extra_tests', (ContainerTypeInfo(dict, (str, File))), @@ -238,7 +208,7 @@ def init_attrs(state: 'ModuleState', def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'FeatureObject': @noPosargs - @typed_kwargs('feature.update', + @typed_kwargs('features.FeatureObject.update', KwargInfo('name', (NoneType, str)), KwargInfo('interest', (NoneType, int)), KwargInfo( @@ -252,8 +222,8 @@ def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], 'group', (NoneType, str, ContainerTypeInfo(list, str)), listify=True ), - KwargConfilctAttr('feature.update', 'detect'), - KwargConfilctAttr('feature.update', 'args'), + KwargConfilctAttr('features.FeatureObject.update', 'detect'), + KwargConfilctAttr('features.FeatureObject.update', 'args'), KwargInfo('test_code', (NoneType, str, File)), KwargInfo( 'extra_tests', ( @@ -273,27 +243,28 @@ def update(state: 'ModuleState', args: T.List['TYPE_var'], return self @noKwargs - @typed_pos_args('feature.get', str) + @typed_pos_args('features.FeatureObject.get', str) def get_method(self, state: 'ModuleState', args: T.Tuple[str], kwargs: 'TYPE_kwargs') -> 'TYPE_var': impl_lst = lambda lst: [v.to_dict() for v in lst] noconv = lambda v: v - dfunc = dict( - name = noconv, - interest = noconv, - group = noconv, - implies = lambda v: [fet.name for fet in sorted(v)], - detect = impl_lst, - args = impl_lst, - test_code = noconv, - extra_tests = noconv, - disable = noconv - ) - cfunc = dfunc.get(args[0]) + dfunc = { + 'name': noconv, + 'interest': noconv, + 'group': noconv, + 'implies': lambda v: [fet.name for fet in sorted(v)], + 'detect': impl_lst, + 'args': impl_lst, + 'test_code': noconv, + 'extra_tests': noconv, + 'disable': noconv + } + cfunc: T.Optional[T.Callable[[str], 'TYPE_var']] = dfunc.get(args[0]) if cfunc is None: raise MesonException(f'Key {args[0]!r} is not in the feature.') - return cfunc(getattr(self, args[0])) + val = getattr(self, args[0]) + return cfunc(val) def get_implicit(self, _caller: T.Set['FeatureObject'] = None ) -> T.Set['FeatureObject']: @@ -306,8 +277,61 @@ def get_implicit(self, _caller: T.Set['FeatureObject'] = None ret = ret.union(sub_fet.get_implicit(_caller)) return ret + @staticmethod + def get_implicit_multi(features: T.Iterable['FeatureObject']) -> T.Set['FeatureObject']: + implies = set().union(*[f.get_implicit() for f in features]) + return implies + + @staticmethod + def get_implicit_combine_multi(features: T.Iterable['FeatureObject']) -> T.Set['FeatureObject']: + return FeatureObject.get_implicit_multi(features).union(features) + + @staticmethod + def sorted_multi(features: T.Iterable[T.Union['FeatureObject', T.Iterable['FeatureObject']]], + reverse: bool = False + ) -> T.List[T.Union['FeatureObject', T.Iterable['FeatureObject']]]: + def sort_cb(k: T.Union[FeatureObject, T.Iterable[FeatureObject]]) -> int: + if isinstance(k, FeatureObject): + return k.interest + # keep prevalent features and erase any implied features + implied_features = FeatureObject.get_implicit_multi(k) + prevalent_features = set(k).difference(implied_features) + if len(prevalent_features) == 0: + # It happens when all features imply each other. + # Set the highest interested feature + return sorted(k)[-1].interest + # multiple features + rank = max(f.interest for f in prevalent_features) + # FIXME: that's not a safe way to increase the rank for + # multi features this why this function isn't considerd + # accurate. + rank += len(prevalent_features) -1 + return rank + return sorted(features, reverse=reverse, key=sort_cb) + + @staticmethod + def features_names(features: T.Iterable[T.Union['FeatureObject', T.Iterable['FeatureObject']]] + ) -> T.List[T.Union[str, T.List[str]]]: + return [ + fet.name if isinstance(fet, FeatureObject) + else [f.name for f in fet] + for fet in features + ] + + def __repr__(self) -> str: + args = ', '.join([ + f'{attr} = {str(getattr(self, attr))}' + for attr in [ + 'group', 'implies', + 'detect', 'args', + 'test_code', 'extra_tests', + 'disable' + ] + ]) + return f'FeatureObject({self.name}, {self.interest}, {args})' + def __hash__(self) -> int: - return hash(str(id(self)) + self.name) + return hash(self.name) def __eq__(self, robj: object) -> bool: if not isinstance(robj, FeatureObject): @@ -329,4 +353,3 @@ def __gt__(self, robj: object) -> T.Any: def __ge__(self, robj: object) -> T.Any: return robj <= self - diff --git a/mesonbuild/modules/features/module.py b/mesonbuild/modules/features/module.py new file mode 100644 index 000000000000..8a07ab04d383 --- /dev/null +++ b/mesonbuild/modules/features/module.py @@ -0,0 +1,714 @@ +# Copyright (c) 2023, NumPy Developers. +# All rights reserved. + +import typing as T +import os + +from ... import mlog, build +from ...compilers import Compiler +from ...mesonlib import File, MesonException +from ...interpreter.type_checking import NoneType +from ...interpreterbase.decorators import ( + noKwargs, KwargInfo, typed_kwargs, typed_pos_args, + ContainerTypeInfo, permittedKwargs +) +from .. import ModuleInfo, NewExtensionModule, ModuleObject +from .feature import FeatureObject, ConflictAttr +from .utils import test_code, get_compiler, generate_hash + +if T.TYPE_CHECKING: + from typing import TypedDict + from ...interpreterbase import TYPE_var, TYPE_kwargs + from .. import ModuleState + from .feature import FeatureKwArgs + + class TestKwArgs(TypedDict): + compiler: T.Optional[Compiler] + force_args: T.Optional[T.List[str]] + anyfet: bool + cached: bool + + class TestResultKwArgs(TypedDict): + target_name: str + prevalent_features: T.List[str] + features: T.List[str] + args: T.List[str] + detect: T.List[str] + defines: T.List[str] + undefines: T.List[str] + is_supported: bool + is_disabled: bool + fail_reason: str + +class TargetsObject(ModuleObject): + def __init__(self) -> None: + super().__init__() + self._targets: T.Dict[ + T.Union[FeatureObject, T.Tuple[FeatureObject, ...]], + T.List[build.StaticLibrary] + ] = {} + self._baseline: T.List[build.StaticLibrary] = [] + self.methods.update({ + 'static_lib': self.static_lib_method, + 'extend': self.extend_method + }) + + def extend_method(self, state: 'ModuleState', + args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs') -> 'TargetsObject': + + @typed_pos_args('feature.TargetsObject.extend', TargetsObject) + @noKwargs + def test_args(state: 'ModuleState', + args: T.Tuple[TargetsObject], + kwargs: 'TYPE_kwargs') -> TargetsObject: + return args[0] + robj: TargetsObject = test_args(state, args, kwargs) + self._baseline.extend(robj._baseline) + for features, robj_targets in robj._targets.items(): + targets: T.List[build.StaticLibrary] = self._targets.setdefault(features, []) + targets += robj_targets + return self + + @typed_pos_args('features.TargetsObject.static_lib', str) + @noKwargs + def static_lib_method(self, state: 'ModuleState', args: T.Tuple[str], + kwargs: 'TYPE_kwargs' + ) -> T.Any: + # The linking order must be based on the lowest interested features, + # to ensures that the linker prioritizes any duplicate weak global symbols + # of the lowest interested features over the highest ones, + # starting with the baseline to avoid any possible crashes due + # to any involved optimizations that may generated based + # on the highest interested features. + link_whole = [] + self._baseline + tcast = T.Union[FeatureObject, T.Tuple[FeatureObject, ...]] + for features in FeatureObject.sorted_multi(self._targets.keys()): + link_whole += self._targets[T.cast(tcast, features)] + if not link_whole: + return [] + static_lib = state._interpreter.func_static_lib( + None, [args[0]], { + 'link_whole': link_whole + } + ) + return static_lib + + def add_baseline_target(self, target: build.StaticLibrary) -> None: + self._baseline.append(target) + + def add_target(self, features: T.Union[FeatureObject, T.List[FeatureObject]], + target: build.StaticLibrary) -> None: + tfeatures = ( + features if isinstance(features, FeatureObject) + else tuple(sorted(features)) + ) + targets: T.List[build.StaticLibrary] = self._targets.setdefault( + tfeatures, T.cast(T.List[build.StaticLibrary], [])) # type: ignore + targets.append(target) + +class Module(NewExtensionModule): + INFO = ModuleInfo('features', '0.1.0') + def __init__(self) -> None: + super().__init__() + self.methods.update({ + 'new': self.new_method, + 'test': self.test_method, + 'implicit': self.implicit_method, + 'implicit_c': self.implicit_c_method, + 'sort': self.sort_method, + 'multi_targets': self.multi_targets_method, + }) + + def new_method(self, state: 'ModuleState', + args: T.List['TYPE_var'], + kwargs: 'TYPE_kwargs') -> FeatureObject: + return FeatureObject(state, args, kwargs) + + def _cache_dict(self, state: 'ModuleState' + ) -> T.Dict[str, 'TestResultKwArgs']: + coredata = state.environment.coredata + attr_name = 'module_features_cache' + if not hasattr(coredata, attr_name): + setattr(coredata, attr_name, {}) + return getattr(coredata, attr_name, {}) + + def _get_cache(self, state: 'ModuleState', key: str + ) -> T.Optional['TestResultKwArgs']: + return self._cache_dict(state).get(key) + + def _set_cache(self, state: 'ModuleState', key: str, + val: 'TestResultKwArgs') -> None: + self._cache_dict(state)[key] = val + + @typed_pos_args('features.test', varargs=FeatureObject, min_varargs=1) + @typed_kwargs('features.test', + KwargInfo('compiler', (NoneType, Compiler)), + KwargInfo('anyfet', bool, default = False), + KwargInfo('cached', bool, default = True), + KwargInfo( + 'force_args', (NoneType, str, ContainerTypeInfo(list, str)), + listify=True + ), + ) + def test_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TestKwArgs' + ) -> T.List[T.Union[bool, 'TestResultKwArgs']]: + + features = args[0] + features_set = set(features) + anyfet = kwargs['anyfet'] + cached = kwargs['cached'] + compiler = kwargs.get('compiler') + if not compiler: + compiler = get_compiler(state) + + force_args = kwargs['force_args'] + if force_args is not None: + # removes in empty strings + force_args = [a for a in force_args if a] + + test_cached, test_result = self.cached_test( + state, features=features_set, + compiler=compiler, + anyfet=anyfet, + cached=cached, + force_args=force_args + ) + if not test_result['is_supported']: + if test_result['is_disabled']: + label = mlog.yellow('disabled') + else: + label = mlog.yellow('Unsupported') + else: + label = mlog.green('Supported') + if anyfet: + unsupported = ' '.join([ + fet.name for fet in sorted(features_set) + if fet.name not in test_result['features'] + ]) + if unsupported: + label = mlog.green(f'Parial support, missing({unsupported})') + + features_names = ' '.join([f.name for f in features]) + log_prefix = f'Test features "{mlog.bold(features_names)}" :' + cached_msg = f'({mlog.blue("cached")})' if test_cached else '' + if not test_result['is_supported']: + mlog.log(log_prefix, label, 'due to', test_result['fail_reason']) + else: + mlog.log(log_prefix, label, cached_msg) + return [test_result['is_supported'], test_result] + + def cached_test(self, state: 'ModuleState', + features: T.Set[FeatureObject], + compiler: 'Compiler', + force_args: T.Optional[T.List[str]], + anyfet: bool, cached: bool, + _caller: T.Optional[T.Set[FeatureObject]] = None + ) -> T.Tuple[bool, 'TestResultKwArgs']: + + if cached: + test_hash = generate_hash( + sorted(features), compiler, + anyfet, force_args + ) + test_result = self._get_cache(state, test_hash) + if test_result is not None: + return True, test_result + + if anyfet: + test_func = self.test_any + else: + test_func = self.test + + test_result = test_func( + state, features=features, + compiler=compiler, + force_args=force_args, + cached=cached, + _caller=_caller + ) + if cached: + self._set_cache(state, test_hash, test_result) + return False, test_result + + def test_any(self, state: 'ModuleState', features: T.Set[FeatureObject], + compiler: 'Compiler', + force_args: T.Optional[T.List[str]], + cached: bool, + # dummy no need for recrusive guard + _caller: T.Optional[T.Set[FeatureObject]] = None, + ) -> 'TestResultKwArgs': + + _, test_any_result = self.cached_test( + state, features=features, + compiler=compiler, + anyfet=False, + cached=cached, + force_args=force_args, + ) + if test_any_result['is_supported']: + return test_any_result + + all_features = sorted(FeatureObject.get_implicit_combine_multi(features)) + features_any = set() + for fet in all_features: + _, test_any_result = self.cached_test( + state, features={fet,}, + compiler=compiler, + cached=cached, + anyfet=False, + force_args=force_args, + ) + if test_any_result['is_supported']: + features_any.add(fet) + + _, test_any_result = self.cached_test( + state, features=features_any, + compiler=compiler, + cached=cached, + anyfet=False, + force_args=force_args, + ) + return test_any_result + + def test(self, state: 'ModuleState', features: T.Set[FeatureObject], + compiler: 'Compiler', + force_args: T.Optional[T.List[str]] = None, + cached: bool = True, + _caller: T.Optional[T.Set[FeatureObject]] = None + ) -> 'TestResultKwArgs': + + implied_features = FeatureObject.get_implicit_multi(features) + all_features = sorted(implied_features.union(features)) + # For multiple features, it important to erase any features + # implied by another to avoid duplicate testing since + # implied features already tested also we use this set to genrate + # unque target name that can be used for multiple targets + # build. + prevalent_features = sorted(features.difference(implied_features)) + if len(prevalent_features) == 0: + # It happens when all features imply each other. + # Set the highest interested feature + prevalent_features = sorted(features)[-1:] + + prevalent_names = [fet.name for fet in prevalent_features] + # prepare the result dict + test_result: 'TestResultKwArgs' = { + 'target_name': '__'.join(prevalent_names), + 'prevalent_features': prevalent_names, + 'features': [fet.name for fet in all_features], + 'args': [], + 'detect': [], + 'defines': [], + 'undefines': [], + 'is_supported': True, + 'is_disabled': False, + 'fail_reason': '', + } + def fail_result(fail_reason: str, is_disabled: bool = False + ) -> 'TestResultKwArgs': + test_result.update({ + 'features': [], + 'args': [], + 'detect': [], + 'defines': [], + 'undefines': [], + 'is_supported': False, + 'is_disabled': is_disabled, + 'fail_reason': fail_reason, + }) + return test_result + + # test any of prevalent features wither they disabled or not + for fet in prevalent_features: + if fet.disable: + return fail_result( + f'{fet.name} is disabled due to "{fet.disable}"', + True + ) + + # since we allows features to imply each other + # items of `features` may part of `implied_features` + if _caller is None: + _caller = set() + _caller = _caller.union(prevalent_features) + predecessor_features = implied_features.difference(_caller) + for fet in sorted(predecessor_features): + _, pred_result = self.cached_test( + state, features={fet,}, + compiler=compiler, + cached=cached, + anyfet=False, + force_args=force_args, + _caller=_caller, + ) + if not pred_result['is_supported']: + reason = f'Implied feature "{fet.name}" ' + pred_disabled = pred_result['is_disabled'] + if pred_disabled: + fail_reason = reason + 'is disabled' + else: + fail_reason = reason + 'is not supported' + return fail_result(fail_reason, pred_disabled) + + for k in ['defines', 'undefines']: + def_values = test_result[k] # type: ignore + pred_values = pred_result[k] # type: ignore + def_values += [v for v in pred_values if v not in def_values] + + # Sort based on the lowest interest to deal with conflict attributes + # when combine all attributes togathers + conflict_attrs = ['detect'] + if force_args is None: + conflict_attrs += ['args'] + else: + test_result['args'] = force_args + + for fet in all_features: + for attr in conflict_attrs: + values: T.List[ConflictAttr] = getattr(fet, attr) + accumulate_values = test_result[attr] # type: ignore + for conflict in values: + if not conflict.match: + accumulate_values.append(conflict.val) + continue + conflict_vals: T.List[str] = [] + # select the acc items based on the match + new_acc: T.List[str] = [] + for acc in accumulate_values: + # not affected by the match so we keep it + if not conflict.match.match(acc): + new_acc.append(acc) + continue + # no filter so we totaly escape it + if not conflict.mfilter: + continue + filter_val = conflict.mfilter.findall(acc) + filter_val = [ + conflict.mjoin.join([i for i in val if i]) + if isinstance(val, tuple) else val + for val in filter_val if val + ] + # no filter match so we totaly escape it + if not filter_val: + continue + conflict_vals.append(conflict.mjoin.join(filter_val)) + new_acc.append(conflict.val + conflict.mjoin.join(conflict_vals)) + test_result[attr] = new_acc # type: ignore + + test_args = compiler.has_multi_arguments + args = test_result['args'] + if args: + supported_args, test_cached = test_args(args, state.environment) + if not supported_args: + return fail_result( + f'Arguments "{", ".join(args)}" are not supported' + ) + + for fet in prevalent_features: + if fet.test_code: + _, tested_code, _ = test_code( + state, compiler, args, fet.test_code + ) + if not tested_code: + return fail_result( + f'Compiler fails against the test code of "{fet.name}"' + ) + + test_result['defines'] += [fet.name] + fet.group + for extra_name, extra_test in fet.extra_tests.items(): + _, tested_code, _ = test_code( + state, compiler, args, extra_test + ) + k = 'defines' if tested_code else 'undefines' + test_result[k].append(extra_name) # type: ignore + return test_result + + @permittedKwargs(build.known_stlib_kwargs | { + 'dispatch', 'baseline', 'prefix', 'cached', 'keep_sort' + }) + @typed_pos_args('features.multi_targets', str, min_varargs=1, varargs=( + str, File, build.CustomTarget, build.CustomTargetIndex, + build.GeneratedList, build.StructuredSources, build.ExtractedObjects, + build.BuildTarget + )) + @typed_kwargs('features.multi_targets', + KwargInfo( + 'dispatch', ( + ContainerTypeInfo(list, (FeatureObject, list)), + ), + default=[] + ), + KwargInfo( + 'baseline', ( + NoneType, + ContainerTypeInfo(list, FeatureObject) + ) + ), + KwargInfo('prefix', str, default=''), + KwargInfo('compiler', (NoneType, Compiler)), + KwargInfo('cached', bool, default = True), + KwargInfo('keep_sort', bool, default = False), + allow_unknown=True + ) + def multi_targets_method(self, state: 'ModuleState', + args: T.Tuple[str], kwargs: 'TYPE_kwargs' + ) -> TargetsObject: + config_name = args[0] + sources = args[1] # type: ignore + dispatch: T.List[T.Union[FeatureObject, T.List[FeatureObject]]] = ( + kwargs.pop('dispatch') # type: ignore + ) + baseline: T.Optional[T.List[FeatureObject]] = ( + kwargs.pop('baseline') # type: ignore + ) + prefix: str = kwargs.pop('prefix') # type: ignore + cached: bool = kwargs.pop('cached') # type: ignore + compiler: T.Optional[Compiler] = kwargs.pop('compiler') # type: ignore + if not compiler: + compiler = get_compiler(state) + + baseline_features : T.Set[FeatureObject] = set() + has_baseline = baseline is not None + if has_baseline: + baseline_features = FeatureObject.get_implicit_combine_multi(baseline) + _, baseline_test_result = self.cached_test( + state, features=set(baseline), + anyfet=True, cached=cached, + compiler=compiler, + force_args=None + ) + + enabled_targets_names: T.List[str] = [] + enabled_targets_features: T.List[T.Union[ + FeatureObject, T.List[FeatureObject] + ]] = [] + enabled_targets_tests: T.List['TestResultKwArgs'] = [] + skipped_targets: T.List[T.Tuple[ + T.Union[FeatureObject, T.List[FeatureObject]], str + ]] = [] + for d in dispatch: + if isinstance(d, FeatureObject): + target = {d,} + is_base_part = d in baseline_features + else: + target = set(d) + is_base_part = all(f in baseline_features for f in d) + + if is_base_part: + skipped_targets.append((d, "part of baseline features")) + continue + _, test_result = self.cached_test( + state=state, features=target, + anyfet=False, cached=cached, + compiler=compiler, + force_args=None + ) + if not test_result['is_supported']: + skipped_targets.append( + (d, test_result['fail_reason']) + ) + continue + target_name = test_result['target_name'] + if target_name in enabled_targets_names: + skipped_targets.append(( + d, f'Dublicate target name "{target_name}"' + )) + continue + enabled_targets_names.append(target_name) + enabled_targets_features.append(d) + enabled_targets_tests.append(test_result) + + if not kwargs.pop('keep_sort'): + enabled_targets_sorted = FeatureObject.sorted_multi(enabled_targets_features, reverse=True) + if enabled_targets_features != enabled_targets_sorted: + log_targets = FeatureObject.features_names(enabled_targets_features) + log_targets_sorted = FeatureObject.features_names(enabled_targets_sorted) + raise MesonException( + 'The enabled dispatch features should be sorted based on the highest interest:\n' + f'Expected: {log_targets_sorted}\n' + f'Got: {log_targets}\n' + 'Note: This validation may not be accurate when dealing with multi-features ' + 'per single target.\n' + 'You can keep the current sort and bypass this validation by passing ' + 'the argument "keep_sort: true".' + ) + + config_path = self.gen_config( + state, + config_name=config_name, + targets=enabled_targets_tests, + prefix=prefix, + has_baseline=has_baseline + ) + mtargets_obj = TargetsObject() + if has_baseline: + mtargets_obj.add_baseline_target( + self.gen_target( + state=state, config_name=config_name, + sources=sources, test_result=baseline_test_result, + prefix=prefix, is_baseline=True, + stlib_kwargs=kwargs + ) + ) + for features_objects, target_test in zip(enabled_targets_features, enabled_targets_tests): + static_lib = self.gen_target( + state=state, config_name=config_name, + sources=sources, test_result=target_test, + prefix=prefix, is_baseline=False, + stlib_kwargs=kwargs + ) + mtargets_obj.add_target(features_objects, static_lib) + + skipped_targets_info: T.List[str] = [] + skipped_tab = ' '*4 + for skipped, reason in skipped_targets: + name = ', '.join( + [skipped.name] if isinstance(skipped, FeatureObject) + else [fet.name for fet in skipped] + ) + skipped_targets_info.append(f'{skipped_tab}"{name}": "{reason}"') + + target_info: T.Callable[[str, 'TestResultKwArgs'], str] = lambda target_name, test_result: ( + f'{skipped_tab}"{target_name}":\n' + '\n'.join([ + f'{skipped_tab*2}"{k}": {v}' + for k, v in test_result.items() + ]) + ) + enabled_targets_info: T.List[str] = [ + target_info(test_result['target_name'], test_result) + for test_result in enabled_targets_tests + ] + if has_baseline: + enabled_targets_info.append(target_info( + f'baseline({baseline_test_result["target_name"]})', + baseline_test_result + )) + enabled_targets_names += ['baseline'] + + mlog.log( + f'Generating multi-targets for "{mlog.bold(config_name)}"', + '\n Enabled targets:', + mlog.green(', '.join(enabled_targets_names)) + ) + mlog.debug( + f'Generating multi-targets for "{config_name}"', + '\n Config path:', config_path, + '\n Enabled targets:', + '\n'+'\n'.join(enabled_targets_info), + '\n Skipped targets:', + '\n'+'\n'.join(skipped_targets_info), + '\n' + ) + return mtargets_obj + + def gen_target(self, state: 'ModuleState', config_name: str, + sources: T.List[T.Union[ + str, File, build.CustomTarget, build.CustomTargetIndex, + build.GeneratedList, build.StructuredSources, build.ExtractedObjects, + build.BuildTarget + ]], + test_result: 'TestResultKwArgs', + prefix: str, is_baseline: bool, + stlib_kwargs: T.Dict[str, T.Any] + ) -> build.StaticLibrary: + + target_name = 'baseline' if is_baseline else test_result['target_name'] + args = [f'-D{prefix}HAVE_{df}' for df in test_result['defines']] + args += test_result['args'] + if is_baseline: + args.append(f'-D{prefix}MTARGETS_BASELINE') + else: + args.append(f'-D{prefix}MTARGETS_CURRENT={target_name}') + stlib_kwargs = stlib_kwargs.copy() + stlib_kwargs.update({ + 'sources': sources, + 'c_args': stlib_kwargs.get('c_args', []) + args, + 'cpp_args': stlib_kwargs.get('cpp_args', []) + args + }) + static_lib: build.StaticLibrary = state._interpreter.func_static_lib( + None, [config_name + '_' + target_name], + stlib_kwargs + ) + return static_lib + + def gen_config(self, state: 'ModuleState', config_name: str, + targets: T.List['TestResultKwArgs'], + prefix: str, has_baseline: bool + ) -> str: + + dispatch_calls: T.List[str] = [] + for test in targets: + c_detect = '&&'.join([ + f'TEST_CB({d})' for d in test['detect'] + ]) + if c_detect: + c_detect = f'({c_detect})' + else: + c_detect = '1' + dispatch_calls.append( + f'{prefix}_MTARGETS_EXPAND(' + f'EXEC_CB({c_detect}, {test["target_name"]}, __VA_ARGS__)' + ')' + ) + + config_file = [ + '/* Autogenerated by the Meson features module. */', + '/* Do not edit, your changes will be lost. */', + '', + f'#undef {prefix}_MTARGETS_EXPAND', + f'#define {prefix}_MTARGETS_EXPAND(X) X', + '', + f'#undef {prefix}MTARGETS_CONF_BASELINE', + f'#define {prefix}MTARGETS_CONF_BASELINE(EXEC_CB, ...) ' + ( + f'{prefix}_MTARGETS_EXPAND(EXEC_CB(__VA_ARGS__))' + if has_baseline else '' + ), + '', + f'#undef {prefix}MTARGETS_CONF_DISPATCH', + f'#define {prefix}MTARGETS_CONF_DISPATCH(TEST_CB, EXEC_CB, ...) \\', + ' \\\n'.join(dispatch_calls), + '', + ] + + build_dir = state.environment.build_dir + sub_dir = state.subdir + if sub_dir: + build_dir = os.path.join(build_dir, sub_dir) + config_path = os.path.abspath(os.path.join(build_dir, config_name)) + + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(config_path, "w", encoding='utf-8') as cout: + cout.write('\n'.join(config_file)) + + return config_path + + @typed_pos_args('features.sort', varargs=FeatureObject, min_varargs=1) + @typed_kwargs('features.sort', + KwargInfo('reverse', bool, default = False), + ) + def sort_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: T.Dict[str, bool] + ) -> T.List[FeatureObject]: + return sorted(args[0], reverse=kwargs['reverse']) + + @typed_pos_args('features.implicit', varargs=FeatureObject, min_varargs=1) + @noKwargs + def implicit_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + + features = args[0] + return sorted(FeatureObject.get_implicit_multi(features)) + + @typed_pos_args('features.implicit', varargs=FeatureObject, min_varargs=1) + @noKwargs + def implicit_c_method(self, state: 'ModuleState', + args: T.Tuple[T.List[FeatureObject]], + kwargs: 'TYPE_kwargs' + ) -> T.List[FeatureObject]: + return sorted(FeatureObject.get_implicit_combine_multi(args[0])) diff --git a/mesonbuild/modules/feature/utils.py b/mesonbuild/modules/features/utils.py similarity index 83% rename from mesonbuild/modules/feature/utils.py rename to mesonbuild/modules/features/utils.py index b681e96c169a..e7f82935d553 100644 --- a/mesonbuild/modules/feature/utils.py +++ b/mesonbuild/modules/features/utils.py @@ -2,6 +2,7 @@ # All rights reserved. import typing as T +import hashlib from ...mesonlib import MesonException, MachineChoice if T.TYPE_CHECKING: @@ -32,3 +33,9 @@ def test_code(state: 'ModuleState', compiler: 'Compiler', ) as p: return p.cached, p.returncode == 0, p.stderr +def generate_hash(*args: T.Any) -> str: + hasher = hashlib.sha1() + test: T.List[bytes] = [] + for a in args: + hasher.update(bytes(str(a), encoding='utf-8')) + return hasher.hexdigest() diff --git a/run_unittests.py b/run_unittests.py old mode 100755 new mode 100644 diff --git a/test cases/features/1 baseline/baseline.c b/test cases/features/1 baseline/baseline.c new file mode 100644 index 000000000000..c208e9bcc1ac --- /dev/null +++ b/test cases/features/1 baseline/baseline.c @@ -0,0 +1,77 @@ +// the headers files of enabled CPU features +#ifdef HAVE_SSE + #include +#endif +#ifdef HAVE_SSE2 + #include +#endif +#ifdef HAVE_SSE3 + #include +#endif +#ifdef HAVE_SSSE3 + #include +#endif +#ifdef HAVE_SSE41 + #include +#endif +#ifdef HAVE_NEON + #include +#endif + +int main() { +#if defined( __i386__ ) || defined(i386) || defined(_M_IX86) || \ + defined(__x86_64__) || defined(__amd64__) || defined(__x86_64) || defined(_M_AMD64) + #ifndef HAVE_SSE + #error "expected SSE to be enabled" + #endif + #ifndef HAVE_SSE2 + #error "expected SSE2 to be enabled" + #endif + #ifndef HAVE_SSE3 + #error "expected SSE3 to be enabled" + #endif +#else + #ifdef HAVE_SSE + #error "expected SSE to be disabled" + #endif + #ifdef HAVE_SSE2 + #error "expected SSE2 to be disabled" + #endif + #ifdef HAVE_SSE3 + #error "expected SSE3 to be disabled" + #endif +#endif + +#if defined(__arm__) + #ifndef HAVE_NEON + #error "expected NEON to be enabled" + #endif +#else + #ifdef HAVE_NEON + #error "expected NEON to be disabled" + #endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) + #ifndef HAVE_NEON_FP16 + #error "expected NEON_FP16 to be enabled" + #endif + #ifndef HAVE_NEON_VFPV4 + #error "expected NEON_VFPV4 to be enabled" + #endif + #ifndef HAVE_ASIMD + #error "expected ASIMD to be enabled" + #endif +#else + #ifdef HAVE_NEON_FP16 + #error "expected NEON_FP16 to be disabled" + #endif + #ifdef HAVE_NEON_VFPV4 + #error "expected NEON_VFPV4 to be disabled" + #endif + #ifdef HAVE_ASIMD + #error "expected ASIMD to be disabled" + #endif +#endif + return 0; +} diff --git a/test cases/features/1 baseline/init_features b/test cases/features/1 baseline/init_features new file mode 120000 index 000000000000..d52da150db76 --- /dev/null +++ b/test cases/features/1 baseline/init_features @@ -0,0 +1 @@ +../init_features \ No newline at end of file diff --git a/test cases/features/1 baseline/meson.build b/test cases/features/1 baseline/meson.build new file mode 100644 index 000000000000..71c4299cf3d4 --- /dev/null +++ b/test cases/features/1 baseline/meson.build @@ -0,0 +1,14 @@ +project('baseline', 'c') +subdir('init_features') + +baseline = mod_features.test(SSE3, NEON, anyfet: true) +message(baseline) +baseline_args = baseline[1]['args'] +foreach def : baseline[1]['defines'] + baseline_args += ['-DHAVE_' + def] +endforeach +add_project_arguments(baseline_args, language: ['c', 'cpp']) + +exe = executable('baseline', 'baseline.c') +test('baseline', exe) + diff --git a/test cases/features/2 multi_targets/dispatch.h b/test cases/features/2 multi_targets/dispatch.h new file mode 100644 index 000000000000..d1e00db0f164 --- /dev/null +++ b/test cases/features/2 multi_targets/dispatch.h @@ -0,0 +1,90 @@ +#ifndef DISPATCH_H_ +#define DISPATCH_H_ + +// the headers files of enabled CPU features +#ifdef HAVE_SSE + #include +#endif +#ifdef HAVE_SSE2 + #include +#endif +#ifdef HAVE_SSE3 + #include +#endif +#ifdef HAVE_SSSE3 + #include +#endif +#ifdef HAVE_SSE41 + #include +#endif +#ifdef HAVE_NEON + #include +#endif + +#if defined( __i386__ ) || defined(i386) || defined(_M_IX86) || \ + defined(__x86_64__) || defined(__amd64__) || defined(__x86_64) || defined(_M_AMD64) + #define TEST_X86 +#elif defined(__aarch64__) || defined(_M_ARM64) + #define TEST_ARM64 +#elif defined(__arm__) + #define TEST_ARM +#endif + +enum { + CPU_SSE = 1, + CPU_SSE2, + CPU_SSE3, + CPU_SSSE3, + CPU_SSE41, + CPU_NEON, + CPU_NEON_FP16, + CPU_NEON_VFPV4, + CPU_ASIMD +}; +int cpu_has(int feature_id); +#define CPU_TEST(FEATURE_NAME) cpu_has(CPU_##FEATURE_NAME) +#define CPU_TEST_DUMMY(FEATURE_NAME) + +#define EXPAND(X) X +#define CAT__(A, B) A ## B +#define CAT_(A, B) CAT__(A, B) +#define CAT(A, B) CAT_(A, B) +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) + +#ifdef MTARGETS_CURRENT + #define DISPATCH_CURRENT(X) CAT(CAT(X,_), MTARGETS_CURRENT) +#else + // baseline + #define DISPATCH_CURRENT(X) X +#endif + +#define DISPATCH_DECLARE(...) \ + MTARGETS_CONF_DISPATCH(CPU_TEST_DUMMY, DISPATCH_DECLARE_CB, __VA_ARGS__) \ + MTARGETS_CONF_BASELINE(DISPATCH_DECLARE_BASE_CB, __VA_ARGS__) + +// Preprocessor callbacks +#define DISPATCH_DECLARE_CB(TESTED_FEATURES_DUMMY, TARGET_NAME, LEFT, ...) \ + CAT(CAT(LEFT, _), TARGET_NAME) __VA_ARGS__; +#define DISPATCH_DECLARE_BASE_CB(LEFT, ...) \ + LEFT __VA_ARGS__; + +#define DISPATCH_CALL(NAME) \ + ( \ + MTARGETS_CONF_DISPATCH(CPU_TEST, DISPATCH_CALL_CB, NAME) \ + MTARGETS_CONF_BASELINE(DISPATCH_CALL_BASE_CB, NAME) \ + NULL \ + ) +// Preprocessor callbacks +#define DISPATCH_CALL_CB(TESTED_FEATURES, TARGET_NAME, LEFT) \ + (TESTED_FEATURES) ? CAT(CAT(LEFT, _), TARGET_NAME) : +#define DISPATCH_CALL_BASE_CB(LEFT) \ + (1) ? LEFT : + +#include "dispatch1.conf.h" +DISPATCH_DECLARE(const char *dispatch1, ()) + +#include "dispatch2.conf.h" +DISPATCH_DECLARE(const char *dispatch2, ()) + +#endif // DISPATCH_H_ diff --git a/test cases/features/2 multi_targets/dispatch1.c b/test cases/features/2 multi_targets/dispatch1.c new file mode 100644 index 000000000000..b4463355e2a5 --- /dev/null +++ b/test cases/features/2 multi_targets/dispatch1.c @@ -0,0 +1,28 @@ +#include "dispatch.h" + +const char *DISPATCH_CURRENT(dispatch1)() +{ +#ifdef HAVE_SSSE3 + #ifndef HAVE_SSE3 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_SSE2 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_SSE + #error "expected a defention for implied features" + #endif +#endif +#ifdef HAVE_ASIMD + #ifndef HAVE_NEON + #error "expected a defention for implied features" + #endif + #ifndef HAVE_NEON_FP16 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_NEON_VFPV4 + #error "expected a defention for implied features" + #endif +#endif + return TOSTRING(DISPATCH_CURRENT(dispatch1)); +} diff --git a/test cases/features/2 multi_targets/dispatch2.c b/test cases/features/2 multi_targets/dispatch2.c new file mode 100644 index 000000000000..7a69fc3d0375 --- /dev/null +++ b/test cases/features/2 multi_targets/dispatch2.c @@ -0,0 +1,21 @@ +#include "dispatch.h" + +const char *DISPATCH_CURRENT(dispatch2)() +{ +#ifdef HAVE_SSE41 + #ifndef HAVE_SSSE3 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_SSE3 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_SSE2 + #error "expected a defention for implied features" + #endif + #ifndef HAVE_SSE + #error "expected a defention for implied features" + #endif +#endif + return TOSTRING(DISPATCH_CURRENT(dispatch2)); +} + diff --git a/test cases/features/2 multi_targets/dispatch3.c b/test cases/features/2 multi_targets/dispatch3.c new file mode 100644 index 000000000000..a3a555af2ca7 --- /dev/null +++ b/test cases/features/2 multi_targets/dispatch3.c @@ -0,0 +1 @@ +#error "no targets enabled for this source" diff --git a/test cases/features/2 multi_targets/init_features b/test cases/features/2 multi_targets/init_features new file mode 120000 index 000000000000..d52da150db76 --- /dev/null +++ b/test cases/features/2 multi_targets/init_features @@ -0,0 +1 @@ +../init_features \ No newline at end of file diff --git a/test cases/features/2 multi_targets/main.c b/test cases/features/2 multi_targets/main.c new file mode 100644 index 000000000000..88ea11ae767c --- /dev/null +++ b/test cases/features/2 multi_targets/main.c @@ -0,0 +1,45 @@ +#include "dispatch.h" +#include +#include + +int cpu_has(int feature_id) +{ + // we assume the used features are supported by CPU + return 1; +} + +int main() +{ + #include "dispatch1.conf.h" + const char *dispatch1_str = DISPATCH_CALL(dispatch1)(); +#if defined(TEST_X86) + const char *exp_dispatch1_str = "dispatch1_SSSE3"; +#elif defined(TEST_ARM64) + const char *exp_dispatch1_str = "dispatch1"; +#elif defined(TEST_ARM) + const char *exp_dispatch1_str = "dispatch1_ASIMD"; +#else + const char *exp_dispatch1_str = "dispatch1"; +#endif + if (strcmp(dispatch1_str, exp_dispatch1_str) != 0) { + return 1; + } + #include "dispatch2.conf.h" + const char *dispatch2_str = DISPATCH_CALL(dispatch2)(); +#if defined(TEST_X86) + const char *exp_dispatch2_str = "dispatch2_SSE41"; +#elif defined(TEST_ARM64) || defined(TEST_ARM) + const char *exp_dispatch2_str = "dispatch2_ASIMD"; +#else + const char *exp_dispatch2_str = "dispatch2"; +#endif + if (strcmp(dispatch2_str, exp_dispatch2_str) != 0) { + return 2; + } + #include "dispatch3.conf.h" + const char *dispatch3_str = DISPATCH_CALL(dispatch3); + if (dispatch3_str != NULL) { + return 3; + } + return 0; +} diff --git a/test cases/features/2 multi_targets/meson.build b/test cases/features/2 multi_targets/meson.build new file mode 100644 index 000000000000..ffceda098f1a --- /dev/null +++ b/test cases/features/2 multi_targets/meson.build @@ -0,0 +1,26 @@ +project('multi_targets', 'c') +subdir('init_features') + +multi_targets = mod_features.multi_targets( + 'dispatch1.conf.h', 'dispatch1.c', + dispatch: [SSSE3, ASIMD], + baseline: [SSE3, NEON] +) + +multi_targets.extend(mod_features.multi_targets( + 'dispatch2.conf.h', 'dispatch2.c', + dispatch: [SSE41, SSSE3, ASIMD], + baseline: [SSE3] +)) + +multi_targets.extend(mod_features.multi_targets( + 'dispatch3.conf.h', 'dispatch3.c', + dispatch: [] +)) + +exe = executable( + 'multi_targets', 'main.c', + link_with: multi_targets.static_lib('multi_targets_lib') +) + +test('multi_targets', exe) diff --git a/test cases/features/init_features/checks/cpu_asimd.c b/test cases/features/init_features/checks/cpu_asimd.c new file mode 100644 index 000000000000..6bc9022a58d3 --- /dev/null +++ b/test cases/features/init_features/checks/cpu_asimd.c @@ -0,0 +1,27 @@ +#ifdef _MSC_VER + #include +#endif +#include + +int main(int argc, char **argv) +{ + float *src = (float*)argv[argc-1]; + float32x4_t v1 = vdupq_n_f32(src[0]), v2 = vdupq_n_f32(src[1]); + /* MAXMIN */ + int ret = (int)vgetq_lane_f32(vmaxnmq_f32(v1, v2), 0); + ret += (int)vgetq_lane_f32(vminnmq_f32(v1, v2), 0); + /* ROUNDING */ + ret += (int)vgetq_lane_f32(vrndq_f32(v1), 0); +#ifdef __aarch64__ + { + double *src2 = (double*)argv[argc-1]; + float64x2_t vd1 = vdupq_n_f64(src2[0]), vd2 = vdupq_n_f64(src2[1]); + /* MAXMIN */ + ret += (int)vgetq_lane_f64(vmaxnmq_f64(vd1, vd2), 0); + ret += (int)vgetq_lane_f64(vminnmq_f64(vd1, vd2), 0); + /* ROUNDING */ + ret += (int)vgetq_lane_f64(vrndq_f64(vd1), 0); + } +#endif + return ret; +} diff --git a/test cases/features/init_features/checks/cpu_neon.c b/test cases/features/init_features/checks/cpu_neon.c new file mode 100644 index 000000000000..8c64f864dea6 --- /dev/null +++ b/test cases/features/init_features/checks/cpu_neon.c @@ -0,0 +1,19 @@ +#ifdef _MSC_VER + #include +#endif +#include + +int main(int argc, char **argv) +{ + // passing from untraced pointers to avoid optimizing out any constants + // so we can test against the linker. + float *src = (float*)argv[argc-1]; + float32x4_t v1 = vdupq_n_f32(src[0]), v2 = vdupq_n_f32(src[1]); + int ret = (int)vgetq_lane_f32(vmulq_f32(v1, v2), 0); +#ifdef __aarch64__ + double *src2 = (double*)argv[argc-2]; + float64x2_t vd1 = vdupq_n_f64(src2[0]), vd2 = vdupq_n_f64(src2[1]); + ret += (int)vgetq_lane_f64(vmulq_f64(vd1, vd2), 0); +#endif + return ret; +} diff --git a/test cases/features/init_features/checks/cpu_neon_fp16.c b/test cases/features/init_features/checks/cpu_neon_fp16.c new file mode 100644 index 000000000000..f3b949770db6 --- /dev/null +++ b/test cases/features/init_features/checks/cpu_neon_fp16.c @@ -0,0 +1,11 @@ +#ifdef _MSC_VER + #include +#endif +#include + +int main(int argc, char **argv) +{ + short *src = (short*)argv[argc-1]; + float32x4_t v_z4 = vcvt_f32_f16((float16x4_t)vld1_s16(src)); + return (int)vgetq_lane_f32(v_z4, 0); +} diff --git a/test cases/features/init_features/checks/cpu_neon_vfpv4.c b/test cases/features/init_features/checks/cpu_neon_vfpv4.c new file mode 100644 index 000000000000..a039159ddeed --- /dev/null +++ b/test cases/features/init_features/checks/cpu_neon_vfpv4.c @@ -0,0 +1,21 @@ +#ifdef _MSC_VER + #include +#endif +#include + +int main(int argc, char **argv) +{ + float *src = (float*)argv[argc-1]; + float32x4_t v1 = vdupq_n_f32(src[0]); + float32x4_t v2 = vdupq_n_f32(src[1]); + float32x4_t v3 = vdupq_n_f32(src[2]); + int ret = (int)vgetq_lane_f32(vfmaq_f32(v1, v2, v3), 0); +#ifdef __aarch64__ + double *src2 = (double*)argv[argc-2]; + float64x2_t vd1 = vdupq_n_f64(src2[0]); + float64x2_t vd2 = vdupq_n_f64(src2[1]); + float64x2_t vd3 = vdupq_n_f64(src2[2]); + ret += (int)vgetq_lane_f64(vfmaq_f64(vd1, vd2, vd3), 0); +#endif + return ret; +} diff --git a/test cases/features/init_features/checks/cpu_sse.c b/test cases/features/init_features/checks/cpu_sse.c new file mode 100644 index 000000000000..bb98bf63c0b9 --- /dev/null +++ b/test cases/features/init_features/checks/cpu_sse.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + __m128 a = _mm_add_ps(_mm_setzero_ps(), _mm_setzero_ps()); + return (int)_mm_cvtss_f32(a); +} diff --git a/test cases/features/init_features/checks/cpu_sse2.c b/test cases/features/init_features/checks/cpu_sse2.c new file mode 100644 index 000000000000..658afc9b4abf --- /dev/null +++ b/test cases/features/init_features/checks/cpu_sse2.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + __m128i a = _mm_add_epi16(_mm_setzero_si128(), _mm_setzero_si128()); + return _mm_cvtsi128_si32(a); +} diff --git a/test cases/features/init_features/checks/cpu_sse3.c b/test cases/features/init_features/checks/cpu_sse3.c new file mode 100644 index 000000000000..aece1e60174c --- /dev/null +++ b/test cases/features/init_features/checks/cpu_sse3.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + __m128 a = _mm_hadd_ps(_mm_setzero_ps(), _mm_setzero_ps()); + return (int)_mm_cvtss_f32(a); +} diff --git a/test cases/features/init_features/checks/cpu_sse41.c b/test cases/features/init_features/checks/cpu_sse41.c new file mode 100644 index 000000000000..bfdb9feacc47 --- /dev/null +++ b/test cases/features/init_features/checks/cpu_sse41.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + __m128 a = _mm_floor_ps(_mm_setzero_ps()); + return (int)_mm_cvtss_f32(a); +} diff --git a/test cases/features/init_features/checks/cpu_ssse3.c b/test cases/features/init_features/checks/cpu_ssse3.c new file mode 100644 index 000000000000..ad0abc1e66fb --- /dev/null +++ b/test cases/features/init_features/checks/cpu_ssse3.c @@ -0,0 +1,7 @@ +#include + +int main(void) +{ + __m128i a = _mm_hadd_epi16(_mm_setzero_si128(), _mm_setzero_si128()); + return (int)_mm_cvtsi128_si32(a); +} diff --git a/test cases/features/init_features/meson.build b/test cases/features/init_features/meson.build new file mode 100644 index 000000000000..c0b4f13fa862 --- /dev/null +++ b/test cases/features/init_features/meson.build @@ -0,0 +1,98 @@ +#project('test-features', 'c') +mod_features = import('features') +cpu_family = host_machine.cpu_family() +compiler_id = meson.get_compiler('c').get_id() +source_root = meson.project_source_root() + '/../init_features/' +# Basic X86 Features +# ------------------ +SSE = mod_features.new( + 'SSE', 1, args: '-msse', + test_code: files(source_root + 'checks/cpu_sse.c')[0] +) +SSE2 = mod_features.new( + 'SSE2', 2, implies: SSE, + args: '-msse2', + test_code: files(source_root + 'checks/cpu_sse2.c')[0] +) +# enabling SSE without SSE2 is useless also +# it's non-optional for x86_64 +SSE.update(implies: SSE2) +SSE3 = mod_features.new( + 'SSE3', 3, implies: SSE2, + args: '-msse3', + test_code: files(source_root + 'checks/cpu_sse3.c')[0] +) +SSSE3 = mod_features.new( + 'SSSE3', 4, implies: SSE3, + args: '-mssse3', + test_code: files(source_root + 'checks/cpu_ssse3.c')[0] +) +SSE41 = mod_features.new( + 'SSE41', 5, implies: SSSE3, + args: '-msse4.1', + test_code: files(source_root + 'checks/cpu_sse41.c')[0] +) +if cpu_family not in ['x86', 'x86_64'] + # should disable any prevalent features + SSE.update(disable: 'not supported by the current platform') +endif +# Specializations for non unix-like compilers +if compiler_id == 'intel-cl' + foreach fet : [SSE, SSE2, SSE3, SSSE3] + fet.update(args: {'val': '/arch:' + fet.get('name'), 'match': '/arch:.*'}) + endforeach + SSE41.update(args: {'val': '/arch:SSE4.1', 'match': '/arch:.*'}) +elif compiler_id == 'msvc' + # only available on 32-bit. Its enabled by default on 64-bit mode + foreach fet : [SSE, SSE2] + if cpu_family == 'x86' + fet.update(args: {'val': '/arch:' + fet.get('name'), 'match': clear_arch}) + else + fet.update(args: '') + endif + endforeach + # The following features don't own private FLAGS still + # the compiler provides ISA capability for them. + foreach fet : [SSE3, SSSE3, SSE41] + fet.update(args: '') + endforeach +endif + +# Basic ARM Features +# ------------------ +NEON = mod_features.new( + 'NEON', 200, + test_code: files(source_root + 'checks/cpu_neon.c')[0] +) +NEON_FP16 = mod_features.new( + 'NEON_FP16', 201, implies: NEON, + test_code: files(source_root + 'checks/cpu_neon_fp16.c')[0] +) +# FMA +NEON_VFPV4 = mod_features.new( + 'NEON_VFPV4', 202, implies: NEON_FP16, + test_code: files(source_root + 'checks/cpu_neon_vfpv4.c')[0] +) +# Advanced SIMD +ASIMD = mod_features.new( + 'ASIMD', 203, implies: NEON_VFPV4, detect: {'val': 'ASIMD', 'match': 'NEON.*'}, + test_code: files(source_root + 'checks/cpu_asimd.c')[0] +) +if cpu_family == 'aarch64' + # hardware baseline, they can't be enabled independently + NEON.update(implies: [NEON_FP16, NEON_VFPV4, ASIMD]) + NEON_FP16.update(implies: [NEON, NEON_VFPV4, ASIMD]) + NEON_VFPV4.update(implies: [NEON, NEON_FP16, ASIMD]) +elif cpu_family == 'arm' + NEON.update(args: '-mfpu=neon') + NEON_FP16.update(args: ['-mfp16-format=ieee', {'val': '-mfpu=neon-fp16', 'match': '-mfpu=.*'}]) + NEON_VFPV4.update(args: {'val': '-mfpu=neon-vfpv4', 'match': '-mfpu=.*'}) + ASIMD.update(args: [ + {'val': '-mfpu=neon-fp-armv8', 'match': '-mfpu=.*'}, + '-march=armv8-a+simd' + ]) +else + # should disable any prevalent features + NEON.update(disable: 'not supported by the current platform') +endif + diff --git a/unittests/featurestests.py b/unittests/featurestests.py new file mode 100644 index 000000000000..a863a9fb93cc --- /dev/null +++ b/unittests/featurestests.py @@ -0,0 +1,300 @@ +# Copyright (c) 2023, NumPy Developers. + +import re +import contextlib +from mesonbuild.interpreter import Interpreter +from mesonbuild.build import Build +from mesonbuild.mparser import FunctionNode, ArgumentNode, Token +from mesonbuild.modules import ModuleState +from mesonbuild.modules.features import Module +from mesonbuild.compilers import Compiler, CompileResult +from mesonbuild.mesonlib import MachineChoice +from mesonbuild.envconfig import MachineInfo + +from .baseplatformtests import BasePlatformTests +from run_tests import get_convincing_fake_env_and_cc + +class FakeCompiler(Compiler): + language = 'c' + + def __init__(self, trap_args = '', trap_code=''): + super().__init__( + ccache=[], exelist=[], version='0.0', + for_machine=MachineChoice.HOST, + info=MachineInfo( + system='linux', cpu_family='x86_64', + cpu='xeon', endian='little', + kernel='linux', subsystem='numpy' + ), + is_cross=True + ) + self.trap_args = trap_args + self.trap_code = trap_code + + def sanity_check(self, work_dir: str, environment: 'Environment') -> None: + pass + + def get_optimization_args(self, optimization_level: str) -> 'T.List[str]': + return [] + + def get_output_args(self, outputname: str) -> 'T.List[str]': + return [] + + def has_multi_arguments(self, args: 'T.List[str]', env: 'Environment') -> 'T.Tuple[bool, bool]': + if self.trap_args: + for a in args: + if re.match(self.trap_args, a): + return False, False + return True, False + + @contextlib.contextmanager + def compile(self, code: 'mesonlib.FileOrString', *args, **kwargs + ) -> 'T.Iterator[T.Optional[CompileResult]]': + if self.trap_code and re.match(self.trap_code, code): + rcode = -1 + else: + rcode = 0 + result = CompileResult(returncode=rcode) + yield result + + @contextlib.contextmanager + def cached_compile(self, code: 'mesonlib.FileOrString', *args, **kwargs + ) -> 'T.Iterator[T.Optional[CompileResult]]': + if self.trap_code and re.match(self.trap_code, code): + rcode = -1 + else: + rcode = 0 + result = CompileResult(returncode=rcode) + yield result + +class FeaturesTests(BasePlatformTests): + def setUp(self): + super().setUp() + env, cc = get_convincing_fake_env_and_cc( + bdir=self.builddir, prefix=self.prefix) + env.machines.target = env.machines.host + + build = Build(env) + interp = Interpreter(build, mock=True) + project = interp.funcs['project'] + filename = 'featurestests.py' + node = FunctionNode( + filename = filename, + lineno = 0, + colno = 0, + end_lineno = 0, + end_colno = 0, + func_name = 'FeaturesTests', + args = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) + ) + project(node, ['Test Module Features'], {'version': '0.1'}) + self.cc = cc + self.state = ModuleState(interp) + self.mod_features = Module() + + def clear_cache(self): + self.mod_features = Module() + + def mod_method(self, name: str, *args, **kwargs): + mth = self.mod_features.methods.get(name) + return mth(self.state, list(args), kwargs) + + def mod_new(self, *args, **kwargs): + return self.mod_method('new', *args, **kwargs) + + def mod_test(self, *args, **kwargs): + return self.mod_method('test', *args, **kwargs) + + def update_feature(self, feature, **kwargs): + feature.update_method(self.state, [], kwargs) + + def check_result(self, features, expected_result, anyfet=False, **kwargs): + is_supported, test_result = self.mod_test( + *features, compiler=FakeCompiler(**kwargs), + cached=False, anyfet=anyfet + ) + test_result = test_result.copy() # to avoid pop cached dict + test_result.pop('fail_reason') + self.assertEqual(is_supported, expected_result['is_supported']) + self.assertEqual(test_result, expected_result) + + def gen_basic_result(self, prevalent, predecessor=[]): + prevalent_names = [fet.name for fet in prevalent] + predecessor_names = [fet.name for fet in predecessor] + features_names = predecessor_names + prevalent_names + return { + 'target_name': '__'.join(prevalent_names), + 'prevalent_features': prevalent_names, + 'features': features_names, + 'args': [f'arg{i}' for i in range(1, len(features_names) + 1)], + 'detect': features_names, + 'defines': features_names, + 'undefines': [], + 'is_supported': True, + 'is_disabled': False + } + + def gen_fail_result(self, prevalent, is_supported=False, + is_disabled = False): + prevalent_names = [fet.name for fet in prevalent] + return { + 'target_name': '__'.join(prevalent_names), + 'prevalent_features': prevalent_names, + 'features': [], + 'args': [], + 'detect': [], + 'defines': [], + 'undefines': [], + 'is_supported': is_supported, + 'is_disabled': is_disabled + } + + def test_happy_path(self): + fet1 = self.mod_new('fet1', 1, args='arg1', test_code='test1') + fet2 = self.mod_new('fet2', 2, implies=fet1, args='arg2', test_code='test2') + fet3 = self.mod_new('fet3', 3, implies=fet2, args='arg3') + fet4 = self.mod_new('fet4', 4, implies=fet3, args='arg4', test_code='test4') + # fet5 doesn't imply fet4 so we can test target with muti prevalent features + fet5 = self.mod_new('fet5', 5, implies=fet3, args='arg5') + fet6 = self.mod_new('fet6', 6, implies=[fet4, fet5], args='arg6') + + # basic test expected the compiler support all operations + for test_features, prevalent, predecessor in [ + ([fet1], [fet1], []), + ([fet2], [fet2], [fet1]), + ([fet2, fet1], [fet2], [fet1]), + ([fet3], [fet3], [fet1, fet2]), + ([fet2, fet3, fet1], [fet3], [fet1, fet2]), + ([fet4, fet5], [fet4, fet5], [fet1, fet2, fet3]), + ([fet5, fet4], [fet4, fet5], [fet1, fet2, fet3]), + ([fet6], [fet6], [fet1, fet2, fet3, fet4, fet5]), + ]: + expected = self.gen_basic_result(prevalent, predecessor) + self.check_result(test_features, expected) + + for test_features, prevalent, trap_args in [ + ([fet1], [fet1], 'arg1'), + ([fet1], [fet1], 'arg1'), + ]: + expected = self.gen_fail_result(prevalent) + self.check_result(test_features, expected, trap_args=trap_args) + + def test_failures(self): + fet1 = self.mod_new('fet1', 1, args='arg1', test_code='test1') + fet2 = self.mod_new('fet2', 2, implies=fet1, args='arg2', test_code='test2') + fet3 = self.mod_new('fet3', 3, implies=fet2, args='arg3', test_code='test3') + fet4 = self.mod_new('fet4', 4, implies=fet3, args='arg4', test_code='test4') + # fet5 doesn't imply fet4 so we can test target with muti features + fet5 = self.mod_new('fet5', 5, implies=fet3, args='arg5', test_code='test5') + + for test_features, prevalent, disable, trap_args, trap_code in [ + # test by trap flags + ([fet1], [fet1], None, 'arg1', None), + ([fet2], [fet2], None, 'arg1', None), + ([fet2, fet1], [fet2], None, 'arg2', None), + ([fet3], [fet3], None, 'arg1', None), + ([fet3, fet2], [fet3], None, 'arg2', None), + ([fet3, fet1], [fet3], None, 'arg3', None), + ([fet3, fet1], [fet3], None, 'arg3', None), + ([fet5, fet4], [fet4, fet5], None, 'arg4', None), + ([fet5, fet4], [fet4, fet5], None, 'arg5', None), + # test by trap test_code + ([fet1], [fet1], None, None, 'test1'), + ([fet2], [fet2], None, None, 'test1'), + ([fet2, fet1], [fet2], None, None, 'test2'), + ([fet3], [fet3], None, None, 'test1'), + ([fet3, fet2], [fet3], None, None, 'test2'), + ([fet3, fet1], [fet3], None, None, 'test3'), + ([fet5, fet4], [fet4, fet5], None, None, 'test4'), + ([fet5, fet4], [fet4, fet5], None, None, 'test5'), + # test by disable feature + ([fet1], [fet1], fet1, None, None), + ([fet2], [fet2], fet1, None, None), + ([fet2, fet1], [fet2], fet2, None, None), + ([fet3], [fet3], fet1, None, None), + ([fet3, fet2], [fet3], fet2, None, None), + ([fet3, fet1], [fet3], fet3, None, None), + ([fet5, fet4], [fet4, fet5], fet4, None, None), + ([fet5, fet4], [fet4, fet5], fet5, None, None), + ]: + if disable: + self.update_feature(disable, disable='test disable') + expected = self.gen_fail_result(prevalent, is_disabled=not not disable) + self.check_result(test_features, expected, trap_args=trap_args, trap_code=trap_code) + + def test_any(self): + fet1 = self.mod_new('fet1', 1, args='arg1', test_code='test1') + fet2 = self.mod_new('fet2', 2, implies=fet1, args='arg2', test_code='test2') + fet3 = self.mod_new('fet3', 3, implies=fet2, args='arg3', test_code='test3') + fet4 = self.mod_new('fet4', 4, implies=fet3, args='arg4', test_code='test4') + # fet5 doesn't imply fet4 so we can test target with muti features + fet5 = self.mod_new('fet5', 5, implies=fet3, args='arg5', test_code='test5') + fet6 = self.mod_new('fet6', 6, implies=[fet4, fet5], args='arg6') + + for test_features, prevalent, predecessor, trap_args in [ + ([fet2], [fet1], [], 'arg2'), + ([fet6], [fet2], [fet1], 'arg3'), + ([fet6], [fet4], [fet1, fet2, fet3], 'arg5'), + ([fet5, fet4], [fet3], [fet1, fet2], 'arg4|arg5'), + ]: + expected = self.gen_basic_result(prevalent, predecessor) + self.check_result(test_features, expected, trap_args=trap_args, anyfet=True) + + def test_conflict_args(self): + fet1 = self.mod_new('fet1', 1, args='arg1', test_code='test1') + fet2 = self.mod_new('fet2', 2, implies=fet1, args='arg2', test_code='test2') + fet3 = self.mod_new('fet3', 3, implies=fet2, args='arg3') + fet4 = self.mod_new('fet4', 4, implies=fet3, args='arg4', test_code='test4') + fet5 = self.mod_new('fet5', 5, implies=fet3, args='arg5', test_code='test5') + fet6 = self.mod_new('fet6', 6, implies=[fet4, fet5], args='arch=xx') + + compiler = FakeCompiler() + for implies, attr, val, expected_vals in [ + ( + [fet3, fet4, fet5], + 'args', {'val':'arch=the_arch', 'match': 'arg.*'}, + ['arch=the_arch'], + ), + ( + [fet5, fet4], + 'args', {'val':'arch=', 'match': 'arg.*', 'mfilter': '[0-9]'}, + ['arch=12345'], + ), + ( + [fet5, fet4], + 'args', {'val':'arch=', 'match': 'arg.*', 'mfilter': '[0-9]', 'mjoin': '+'}, + ['arch=1+2+3+4+5'], + ), + ( + [fet5, fet4], + 'args', {'val':'arch=num*', 'match': 'arg.*[0-3]', 'mfilter': '[0-9]', 'mjoin': '*'}, + ['arg4', 'arg5', 'arch=num*1*2*3'], + ), + ( + [fet6], + 'args', {'val':'arch=', 'match': 'arg.*[0-9]|arch=.*', 'mfilter': '([0-9])|arch=(\w+)', 'mjoin': '*'}, + ['arch=1*2*3*4*5*xx'], + ), + ( + [fet3, fet4, fet5], + 'detect', {'val':'test_fet', 'match': 'fet.*[0-5]'}, + ['test_fet'], + ), + ( + [fet5, fet4], + 'detect', {'val':'fet', 'match': 'fet.*[0-5]', 'mfilter': '[0-9]'}, + ['fet12345'], + ), + ( + [fet5, fet4], + 'detect', {'val':'fet_', 'match': 'fet.*[0-5]', 'mfilter': '[0-9]', 'mjoin':'_'}, + ['fet_1_2_3_4_5'], + ), + ]: + test_fet = self.mod_new('test_fet', 7, implies=implies, **{attr:val}) + is_supported, test_result = self.mod_test( + test_fet, compiler=compiler, cached=False + ) + self.assertEqual(test_result['is_supported'], True) + self.assertEqual(test_result[attr], expected_vals) + From 248a3d5c59c0202b9e3ebd30464bc2c1ea192318 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 7 Aug 2023 14:24:40 +0400 Subject: [PATCH 11/25] fix initialize Compiler when user_defined_options is None --- mesonbuild/interpreter/interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/interpreter/interpreter.py b/mesonbuild/interpreter/interpreter.py index eb82c9cd137a..fe66f5318091 100644 --- a/mesonbuild/interpreter/interpreter.py +++ b/mesonbuild/interpreter/interpreter.py @@ -1135,7 +1135,7 @@ def set_backend(self) -> None: return from ..backend import backends - if OptionKey('genvslite') in self.user_defined_options.cmd_line_options: + if self.user_defined_options and OptionKey('genvslite') in self.user_defined_options.cmd_line_options: # Use of the '--genvslite vsxxxx' option ultimately overrides any '--backend xxx' # option the user may specify. backend_name = self.coredata.optstore.get_value_for(OptionKey('genvslite')) From fac1e42fda90361e00cde4759722ec67fbf2f2eb Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 7 Aug 2023 17:14:25 +0400 Subject: [PATCH 12/25] expose CompileResult because its needed by unittest --- mesonbuild/compilers/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mesonbuild/compilers/__init__.py b/mesonbuild/compilers/__init__.py index f645090e195b..6c307affe5f8 100644 --- a/mesonbuild/compilers/__init__.py +++ b/mesonbuild/compilers/__init__.py @@ -4,6 +4,7 @@ # Public symbols for compilers sub-package when using 'from . import compilers' __all__ = [ 'Compiler', + 'CompileResult', 'RunResult', 'all_languages', @@ -47,6 +48,7 @@ # Bring symbols from each module into compilers sub-package namespace from .compilers import ( Compiler, + CompileResult, RunResult, all_languages, clib_langs, From 3c0a21de42b3fdc60524b76c6dc9ffb53d8258a6 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 7 Aug 2023 18:57:30 +0400 Subject: [PATCH 13/25] Avoid full importing of module typing --- mesonbuild/modules/features/__init__.py | 6 +- mesonbuild/modules/features/feature.py | 95 ++++++++-------- mesonbuild/modules/features/module.py | 140 ++++++++++++------------ mesonbuild/modules/features/utils.py | 16 ++- 4 files changed, 127 insertions(+), 130 deletions(-) diff --git a/mesonbuild/modules/features/__init__.py b/mesonbuild/modules/features/__init__.py index 3b56affcfbe3..cd71ca43e738 100644 --- a/mesonbuild/modules/features/__init__.py +++ b/mesonbuild/modules/features/__init__.py @@ -1,12 +1,10 @@ # Copyright (c) 2023, NumPy Developers. -# All rights reserved. -# -import typing as T +from typing import TYPE_CHECKING from .module import Module -if T.TYPE_CHECKING: +if TYPE_CHECKING: from ...interpreter import Interpreter def initialize(interpreter: 'Interpreter') -> Module: diff --git a/mesonbuild/modules/features/feature.py b/mesonbuild/modules/features/feature.py index f4021484e28e..7e0f621e543f 100644 --- a/mesonbuild/modules/features/feature.py +++ b/mesonbuild/modules/features/feature.py @@ -1,10 +1,11 @@ # Copyright (c) 2023, NumPy Developers. # All rights reserved. - -import typing as T import re +from typing import ( + Dict, Set, Tuple, List, Callable, Optional, + Union, Any, Iterable, cast, TYPE_CHECKING +) from dataclasses import dataclass, field - from ...mesonlib import File, MesonException from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( @@ -13,7 +14,7 @@ ) from .. import ModuleObject -if T.TYPE_CHECKING: +if TYPE_CHECKING: from typing import TypedDict from typing_extensions import NotRequired from ...interpreterbase import TYPE_var, TYPE_kwargs @@ -40,10 +41,10 @@ class ConflictAttr: """ val: str = field(hash=True, compare=True) - match: T.Union[re.Pattern, None] = field( + match: Union[re.Pattern, None] = field( default=None, hash=False, compare=False ) - mfilter: T.Union[re.Pattern, None] = field( + mfilter: Union[re.Pattern, None] = field( default=None, hash=False, compare=False ) mjoin: str = field(default='', hash=False, compare=False) @@ -51,8 +52,8 @@ class ConflictAttr: def copy(self) -> 'ConflictAttr': return ConflictAttr(**self.__dict__) - def to_dict(self) -> T.Dict[str, str]: - ret: T.Dict[str, str] = {} + def to_dict(self) -> Dict[str, str]: + ret: Dict[str, str] = {} for attr in ('val', 'mjoin'): ret[attr] = getattr(self, attr) for attr in ('match', 'mfilter'): @@ -65,7 +66,7 @@ def to_dict(self) -> T.Dict[str, str]: return ret class KwargConfilctAttr(KwargInfo): - def __init__(self, func_name: str, opt_name: str, default: T.Any = None): + def __init__(self, func_name: str, opt_name: str, default: Any = None): types = ( NoneType, str, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, (dict, str)) @@ -80,10 +81,10 @@ def __init__(self, func_name: str, opt_name: str, default: T.Any = None): @staticmethod def convert(func_name:str, opt_name: str, values: 'IMPLIED_ATTR', - ) -> T.Union[None, T.List[ConflictAttr]]: + ) -> Union[None, List[ConflictAttr]]: if values is None: return None - ret: T.List[ConflictAttr] = [] + ret: List[ConflictAttr] = [] values = [values] if isinstance(values, (str, dict)) else values accepted_keys = ('val', 'match', 'mfilter', 'mjoin') for edict in values: @@ -124,20 +125,20 @@ def convert(func_name:str, opt_name: str, values: 'IMPLIED_ATTR', ret.append(implattr) return ret -if T.TYPE_CHECKING: - IMPLIED_ATTR = T.Union[ - None, str, T.Dict[str, str], T.List[ - T.Union[str, T.Dict[str, str]] +if TYPE_CHECKING: + IMPLIED_ATTR = Union[ + None, str, Dict[str, str], List[ + Union[str, Dict[str, str]] ] ] class FeatureKwArgs(TypedDict): - #implies: T.Optional[T.List['FeatureObject']] - implies: NotRequired[T.List[T.Any]] - group: NotRequired[T.List[str]] - detect: NotRequired[T.List[ConflictAttr]] - args: NotRequired[T.List[ConflictAttr]] - test_code: NotRequired[T.Union[str, File]] - extra_tests: NotRequired[T.Dict[str, T.Union[str, File]]] + #implies: Optional[List['FeatureObject']] + implies: NotRequired[List[Any]] + group: NotRequired[List[str]] + detect: NotRequired[List[ConflictAttr]] + args: NotRequired[List[ConflictAttr]] + test_code: NotRequired[Union[str, File]] + extra_tests: NotRequired[Dict[str, Union[str, File]]] disable: NotRequired[str] class FeatureUpdateKwArgs(FeatureKwArgs): @@ -147,16 +148,16 @@ class FeatureUpdateKwArgs(FeatureKwArgs): class FeatureObject(ModuleObject): name: str interest: int - implies: T.Set['FeatureObject'] - group: T.List[str] - detect: T.List[ConflictAttr] - args: T.List[ConflictAttr] - test_code: T.Union[str, File] - extra_tests: T.Dict[str, T.Union[str, File]] + implies: Set['FeatureObject'] + group: List[str] + detect: List[ConflictAttr] + args: List[ConflictAttr] + test_code: Union[str, File] + extra_tests: Dict[str, Union[str, File]] disable: str def __init__(self, state: 'ModuleState', - args: T.List['TYPE_var'], + args: List['TYPE_var'], kwargs: 'TYPE_kwargs') -> None: super().__init__() @@ -182,7 +183,7 @@ def __init__(self, state: 'ModuleState', KwargInfo('disable', (str), default=''), ) def init_attrs(state: 'ModuleState', - args: T.Tuple[str, int], + args: Tuple[str, int], kwargs: 'FeatureKwArgs') -> None: self.name = args[0] self.interest = args[1] @@ -205,7 +206,7 @@ def init_attrs(state: 'ModuleState', 'get': self.get_method, }) - def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], + def update_method(self, state: 'ModuleState', args: List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'FeatureObject': @noPosargs @typed_kwargs('features.FeatureObject.update', @@ -231,7 +232,7 @@ def update_method(self, state: 'ModuleState', args: T.List['TYPE_var'], ), KwargInfo('disable', (NoneType, str)), ) - def update(state: 'ModuleState', args: T.List['TYPE_var'], + def update(state: 'ModuleState', args: List['TYPE_var'], kwargs: 'FeatureUpdateKwArgs') -> None: for k, v in kwargs.items(): if v is not None and k != 'implies': @@ -244,7 +245,7 @@ def update(state: 'ModuleState', args: T.List['TYPE_var'], @noKwargs @typed_pos_args('features.FeatureObject.get', str) - def get_method(self, state: 'ModuleState', args: T.Tuple[str], + def get_method(self, state: 'ModuleState', args: Tuple[str], kwargs: 'TYPE_kwargs') -> 'TYPE_var': impl_lst = lambda lst: [v.to_dict() for v in lst] @@ -260,14 +261,14 @@ def get_method(self, state: 'ModuleState', args: T.Tuple[str], 'extra_tests': noconv, 'disable': noconv } - cfunc: T.Optional[T.Callable[[str], 'TYPE_var']] = dfunc.get(args[0]) + cfunc: Optional[Callable[[str], 'TYPE_var']] = dfunc.get(args[0]) if cfunc is None: raise MesonException(f'Key {args[0]!r} is not in the feature.') val = getattr(self, args[0]) return cfunc(val) - def get_implicit(self, _caller: T.Set['FeatureObject'] = None - ) -> T.Set['FeatureObject']: + def get_implicit(self, _caller: Set['FeatureObject'] = None + ) -> Set['FeatureObject']: # infinity recursive guard since # features can imply each other _caller = {self, } if not _caller else _caller.union({self, }) @@ -278,19 +279,19 @@ def get_implicit(self, _caller: T.Set['FeatureObject'] = None return ret @staticmethod - def get_implicit_multi(features: T.Iterable['FeatureObject']) -> T.Set['FeatureObject']: + def get_implicit_multi(features: Iterable['FeatureObject']) -> Set['FeatureObject']: implies = set().union(*[f.get_implicit() for f in features]) return implies @staticmethod - def get_implicit_combine_multi(features: T.Iterable['FeatureObject']) -> T.Set['FeatureObject']: + def get_implicit_combine_multi(features: Iterable['FeatureObject']) -> Set['FeatureObject']: return FeatureObject.get_implicit_multi(features).union(features) @staticmethod - def sorted_multi(features: T.Iterable[T.Union['FeatureObject', T.Iterable['FeatureObject']]], + def sorted_multi(features: Iterable[Union['FeatureObject', Iterable['FeatureObject']]], reverse: bool = False - ) -> T.List[T.Union['FeatureObject', T.Iterable['FeatureObject']]]: - def sort_cb(k: T.Union[FeatureObject, T.Iterable[FeatureObject]]) -> int: + ) -> List[Union['FeatureObject', Iterable['FeatureObject']]]: + def sort_cb(k: Union[FeatureObject, Iterable[FeatureObject]]) -> int: if isinstance(k, FeatureObject): return k.interest # keep prevalent features and erase any implied features @@ -310,8 +311,8 @@ def sort_cb(k: T.Union[FeatureObject, T.Iterable[FeatureObject]]) -> int: return sorted(features, reverse=reverse, key=sort_cb) @staticmethod - def features_names(features: T.Iterable[T.Union['FeatureObject', T.Iterable['FeatureObject']]] - ) -> T.List[T.Union[str, T.List[str]]]: + def features_names(features: Iterable[Union['FeatureObject', Iterable['FeatureObject']]] + ) -> List[Union[str, List[str]]]: return [ fet.name if isinstance(fet, FeatureObject) else [f.name for f in fet] @@ -338,18 +339,18 @@ def __eq__(self, robj: object) -> bool: return False return self is robj and self.name == robj.name - def __lt__(self, robj: object) -> T.Any: + def __lt__(self, robj: object) -> Any: if not isinstance(robj, FeatureObject): return NotImplemented return self.interest < robj.interest - def __le__(self, robj: object) -> T.Any: + def __le__(self, robj: object) -> Any: if not isinstance(robj, FeatureObject): return NotImplemented return self.interest <= robj.interest - def __gt__(self, robj: object) -> T.Any: + def __gt__(self, robj: object) -> Any: return robj < self - def __ge__(self, robj: object) -> T.Any: + def __ge__(self, robj: object) -> Any: return robj <= self diff --git a/mesonbuild/modules/features/module.py b/mesonbuild/modules/features/module.py index 8a07ab04d383..0be6af06c8b7 100644 --- a/mesonbuild/modules/features/module.py +++ b/mesonbuild/modules/features/module.py @@ -1,9 +1,9 @@ # Copyright (c) 2023, NumPy Developers. -# All rights reserved. - -import typing as T import os - +from typing import ( + Dict, Set, Tuple, List, Callable, Optional, + Union, Any, cast, TYPE_CHECKING +) from ... import mlog, build from ...compilers import Compiler from ...mesonlib import File, MesonException @@ -16,26 +16,26 @@ from .feature import FeatureObject, ConflictAttr from .utils import test_code, get_compiler, generate_hash -if T.TYPE_CHECKING: +if TYPE_CHECKING: from typing import TypedDict from ...interpreterbase import TYPE_var, TYPE_kwargs from .. import ModuleState from .feature import FeatureKwArgs class TestKwArgs(TypedDict): - compiler: T.Optional[Compiler] - force_args: T.Optional[T.List[str]] + compiler: Optional[Compiler] + force_args: Optional[List[str]] anyfet: bool cached: bool class TestResultKwArgs(TypedDict): target_name: str - prevalent_features: T.List[str] - features: T.List[str] - args: T.List[str] - detect: T.List[str] - defines: T.List[str] - undefines: T.List[str] + prevalent_features: List[str] + features: List[str] + args: List[str] + detect: List[str] + defines: List[str] + undefines: List[str] is_supported: bool is_disabled: bool fail_reason: str @@ -43,38 +43,38 @@ class TestResultKwArgs(TypedDict): class TargetsObject(ModuleObject): def __init__(self) -> None: super().__init__() - self._targets: T.Dict[ - T.Union[FeatureObject, T.Tuple[FeatureObject, ...]], - T.List[build.StaticLibrary] + self._targets: Dict[ + Union[FeatureObject, Tuple[FeatureObject, ...]], + List[build.StaticLibrary] ] = {} - self._baseline: T.List[build.StaticLibrary] = [] + self._baseline: List[build.StaticLibrary] = [] self.methods.update({ 'static_lib': self.static_lib_method, 'extend': self.extend_method }) def extend_method(self, state: 'ModuleState', - args: T.List['TYPE_var'], + args: List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'TargetsObject': @typed_pos_args('feature.TargetsObject.extend', TargetsObject) @noKwargs def test_args(state: 'ModuleState', - args: T.Tuple[TargetsObject], + args: Tuple[TargetsObject], kwargs: 'TYPE_kwargs') -> TargetsObject: return args[0] robj: TargetsObject = test_args(state, args, kwargs) self._baseline.extend(robj._baseline) for features, robj_targets in robj._targets.items(): - targets: T.List[build.StaticLibrary] = self._targets.setdefault(features, []) + targets: List[build.StaticLibrary] = self._targets.setdefault(features, []) targets += robj_targets return self @typed_pos_args('features.TargetsObject.static_lib', str) @noKwargs - def static_lib_method(self, state: 'ModuleState', args: T.Tuple[str], + def static_lib_method(self, state: 'ModuleState', args: Tuple[str], kwargs: 'TYPE_kwargs' - ) -> T.Any: + ) -> Any: # The linking order must be based on the lowest interested features, # to ensures that the linker prioritizes any duplicate weak global symbols # of the lowest interested features over the highest ones, @@ -82,9 +82,9 @@ def static_lib_method(self, state: 'ModuleState', args: T.Tuple[str], # to any involved optimizations that may generated based # on the highest interested features. link_whole = [] + self._baseline - tcast = T.Union[FeatureObject, T.Tuple[FeatureObject, ...]] + tcast = Union[FeatureObject, Tuple[FeatureObject, ...]] for features in FeatureObject.sorted_multi(self._targets.keys()): - link_whole += self._targets[T.cast(tcast, features)] + link_whole += self._targets[cast(tcast, features)] if not link_whole: return [] static_lib = state._interpreter.func_static_lib( @@ -97,14 +97,14 @@ def static_lib_method(self, state: 'ModuleState', args: T.Tuple[str], def add_baseline_target(self, target: build.StaticLibrary) -> None: self._baseline.append(target) - def add_target(self, features: T.Union[FeatureObject, T.List[FeatureObject]], + def add_target(self, features: Union[FeatureObject, List[FeatureObject]], target: build.StaticLibrary) -> None: tfeatures = ( features if isinstance(features, FeatureObject) else tuple(sorted(features)) ) - targets: T.List[build.StaticLibrary] = self._targets.setdefault( - tfeatures, T.cast(T.List[build.StaticLibrary], [])) # type: ignore + targets: List[build.StaticLibrary] = self._targets.setdefault( + tfeatures, cast(List[build.StaticLibrary], [])) # type: ignore targets.append(target) class Module(NewExtensionModule): @@ -121,12 +121,12 @@ def __init__(self) -> None: }) def new_method(self, state: 'ModuleState', - args: T.List['TYPE_var'], + args: List['TYPE_var'], kwargs: 'TYPE_kwargs') -> FeatureObject: return FeatureObject(state, args, kwargs) def _cache_dict(self, state: 'ModuleState' - ) -> T.Dict[str, 'TestResultKwArgs']: + ) -> Dict[str, 'TestResultKwArgs']: coredata = state.environment.coredata attr_name = 'module_features_cache' if not hasattr(coredata, attr_name): @@ -134,7 +134,7 @@ def _cache_dict(self, state: 'ModuleState' return getattr(coredata, attr_name, {}) def _get_cache(self, state: 'ModuleState', key: str - ) -> T.Optional['TestResultKwArgs']: + ) -> Optional['TestResultKwArgs']: return self._cache_dict(state).get(key) def _set_cache(self, state: 'ModuleState', key: str, @@ -152,9 +152,9 @@ def _set_cache(self, state: 'ModuleState', key: str, ), ) def test_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], + args: Tuple[List[FeatureObject]], kwargs: 'TestKwArgs' - ) -> T.List[T.Union[bool, 'TestResultKwArgs']]: + ) -> List[Union[bool, 'TestResultKwArgs']]: features = args[0] features_set = set(features) @@ -201,12 +201,12 @@ def test_method(self, state: 'ModuleState', return [test_result['is_supported'], test_result] def cached_test(self, state: 'ModuleState', - features: T.Set[FeatureObject], + features: Set[FeatureObject], compiler: 'Compiler', - force_args: T.Optional[T.List[str]], + force_args: Optional[List[str]], anyfet: bool, cached: bool, - _caller: T.Optional[T.Set[FeatureObject]] = None - ) -> T.Tuple[bool, 'TestResultKwArgs']: + _caller: Optional[Set[FeatureObject]] = None + ) -> Tuple[bool, 'TestResultKwArgs']: if cached: test_hash = generate_hash( @@ -233,12 +233,12 @@ def cached_test(self, state: 'ModuleState', self._set_cache(state, test_hash, test_result) return False, test_result - def test_any(self, state: 'ModuleState', features: T.Set[FeatureObject], + def test_any(self, state: 'ModuleState', features: Set[FeatureObject], compiler: 'Compiler', - force_args: T.Optional[T.List[str]], + force_args: Optional[List[str]], cached: bool, # dummy no need for recrusive guard - _caller: T.Optional[T.Set[FeatureObject]] = None, + _caller: Optional[Set[FeatureObject]] = None, ) -> 'TestResultKwArgs': _, test_any_result = self.cached_test( @@ -273,11 +273,11 @@ def test_any(self, state: 'ModuleState', features: T.Set[FeatureObject], ) return test_any_result - def test(self, state: 'ModuleState', features: T.Set[FeatureObject], + def test(self, state: 'ModuleState', features: Set[FeatureObject], compiler: 'Compiler', - force_args: T.Optional[T.List[str]] = None, + force_args: Optional[List[str]] = None, cached: bool = True, - _caller: T.Optional[T.Set[FeatureObject]] = None + _caller: Optional[Set[FeatureObject]] = None ) -> 'TestResultKwArgs': implied_features = FeatureObject.get_implicit_multi(features) @@ -368,15 +368,15 @@ def fail_result(fail_reason: str, is_disabled: bool = False for fet in all_features: for attr in conflict_attrs: - values: T.List[ConflictAttr] = getattr(fet, attr) + values: List[ConflictAttr] = getattr(fet, attr) accumulate_values = test_result[attr] # type: ignore for conflict in values: if not conflict.match: accumulate_values.append(conflict.val) continue - conflict_vals: T.List[str] = [] + conflict_vals: List[str] = [] # select the acc items based on the match - new_acc: T.List[str] = [] + new_acc: List[str] = [] for acc in accumulate_values: # not affected by the match so we keep it if not conflict.match.match(acc): @@ -454,23 +454,23 @@ def fail_result(fail_reason: str, is_disabled: bool = False allow_unknown=True ) def multi_targets_method(self, state: 'ModuleState', - args: T.Tuple[str], kwargs: 'TYPE_kwargs' + args: Tuple[str], kwargs: 'TYPE_kwargs' ) -> TargetsObject: config_name = args[0] sources = args[1] # type: ignore - dispatch: T.List[T.Union[FeatureObject, T.List[FeatureObject]]] = ( + dispatch: List[Union[FeatureObject, List[FeatureObject]]] = ( kwargs.pop('dispatch') # type: ignore ) - baseline: T.Optional[T.List[FeatureObject]] = ( + baseline: Optional[List[FeatureObject]] = ( kwargs.pop('baseline') # type: ignore ) prefix: str = kwargs.pop('prefix') # type: ignore cached: bool = kwargs.pop('cached') # type: ignore - compiler: T.Optional[Compiler] = kwargs.pop('compiler') # type: ignore + compiler: Optional[Compiler] = kwargs.pop('compiler') # type: ignore if not compiler: compiler = get_compiler(state) - baseline_features : T.Set[FeatureObject] = set() + baseline_features : Set[FeatureObject] = set() has_baseline = baseline is not None if has_baseline: baseline_features = FeatureObject.get_implicit_combine_multi(baseline) @@ -481,13 +481,13 @@ def multi_targets_method(self, state: 'ModuleState', force_args=None ) - enabled_targets_names: T.List[str] = [] - enabled_targets_features: T.List[T.Union[ - FeatureObject, T.List[FeatureObject] + enabled_targets_names: List[str] = [] + enabled_targets_features: List[Union[ + FeatureObject, List[FeatureObject] ]] = [] - enabled_targets_tests: T.List['TestResultKwArgs'] = [] - skipped_targets: T.List[T.Tuple[ - T.Union[FeatureObject, T.List[FeatureObject]], str + enabled_targets_tests: List['TestResultKwArgs'] = [] + skipped_targets: List[Tuple[ + Union[FeatureObject, List[FeatureObject]], str ]] = [] for d in dispatch: if isinstance(d, FeatureObject): @@ -562,7 +562,7 @@ def multi_targets_method(self, state: 'ModuleState', ) mtargets_obj.add_target(features_objects, static_lib) - skipped_targets_info: T.List[str] = [] + skipped_targets_info: List[str] = [] skipped_tab = ' '*4 for skipped, reason in skipped_targets: name = ', '.join( @@ -571,13 +571,13 @@ def multi_targets_method(self, state: 'ModuleState', ) skipped_targets_info.append(f'{skipped_tab}"{name}": "{reason}"') - target_info: T.Callable[[str, 'TestResultKwArgs'], str] = lambda target_name, test_result: ( + target_info: Callable[[str, 'TestResultKwArgs'], str] = lambda target_name, test_result: ( f'{skipped_tab}"{target_name}":\n' + '\n'.join([ f'{skipped_tab*2}"{k}": {v}' for k, v in test_result.items() ]) ) - enabled_targets_info: T.List[str] = [ + enabled_targets_info: List[str] = [ target_info(test_result['target_name'], test_result) for test_result in enabled_targets_tests ] @@ -605,14 +605,14 @@ def multi_targets_method(self, state: 'ModuleState', return mtargets_obj def gen_target(self, state: 'ModuleState', config_name: str, - sources: T.List[T.Union[ + sources: List[Union[ str, File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList, build.StructuredSources, build.ExtractedObjects, build.BuildTarget ]], test_result: 'TestResultKwArgs', prefix: str, is_baseline: bool, - stlib_kwargs: T.Dict[str, T.Any] + stlib_kwargs: Dict[str, Any] ) -> build.StaticLibrary: target_name = 'baseline' if is_baseline else test_result['target_name'] @@ -635,11 +635,11 @@ def gen_target(self, state: 'ModuleState', config_name: str, return static_lib def gen_config(self, state: 'ModuleState', config_name: str, - targets: T.List['TestResultKwArgs'], + targets: List['TestResultKwArgs'], prefix: str, has_baseline: bool ) -> str: - dispatch_calls: T.List[str] = [] + dispatch_calls: List[str] = [] for test in targets: c_detect = '&&'.join([ f'TEST_CB({d})' for d in test['detect'] @@ -690,17 +690,17 @@ def gen_config(self, state: 'ModuleState', config_name: str, KwargInfo('reverse', bool, default = False), ) def sort_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], - kwargs: T.Dict[str, bool] - ) -> T.List[FeatureObject]: + args: Tuple[List[FeatureObject]], + kwargs: Dict[str, bool] + ) -> List[FeatureObject]: return sorted(args[0], reverse=kwargs['reverse']) @typed_pos_args('features.implicit', varargs=FeatureObject, min_varargs=1) @noKwargs def implicit_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], + args: Tuple[List[FeatureObject]], kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: + ) -> List[FeatureObject]: features = args[0] return sorted(FeatureObject.get_implicit_multi(features)) @@ -708,7 +708,7 @@ def implicit_method(self, state: 'ModuleState', @typed_pos_args('features.implicit', varargs=FeatureObject, min_varargs=1) @noKwargs def implicit_c_method(self, state: 'ModuleState', - args: T.Tuple[T.List[FeatureObject]], + args: Tuple[List[FeatureObject]], kwargs: 'TYPE_kwargs' - ) -> T.List[FeatureObject]: + ) -> List[FeatureObject]: return sorted(FeatureObject.get_implicit_combine_multi(args[0])) diff --git a/mesonbuild/modules/features/utils.py b/mesonbuild/modules/features/utils.py index e7f82935d553..74cb45d8648b 100644 --- a/mesonbuild/modules/features/utils.py +++ b/mesonbuild/modules/features/utils.py @@ -1,11 +1,9 @@ # Copyright (c) 2023, NumPy Developers. -# All rights reserved. - -import typing as T import hashlib +from typing import Tuple, List, Union, Any, TYPE_CHECKING from ...mesonlib import MesonException, MachineChoice -if T.TYPE_CHECKING: +if TYPE_CHECKING: from ...compilers import Compiler from ...mesonlib import File from .. import ModuleState @@ -25,17 +23,17 @@ def get_compiler(state: 'ModuleState') -> 'Compiler': return compiler def test_code(state: 'ModuleState', compiler: 'Compiler', - args: T.List[str], code: 'T.Union[str, File]' - ) -> T.Tuple[bool, bool, str]: - # TODO: treat warnings as errors + args: List[str], code: 'Union[str, File]' + ) -> Tuple[bool, bool, str]: + # TODO: Add option to treat warnings as errors with compiler.cached_compile( code, state.environment.coredata, extra_args=args ) as p: return p.cached, p.returncode == 0, p.stderr -def generate_hash(*args: T.Any) -> str: +def generate_hash(*args: Any) -> str: hasher = hashlib.sha1() - test: T.List[bytes] = [] + test: List[bytes] = [] for a in args: hasher.update(bytes(str(a), encoding='utf-8')) return hasher.hexdigest() From f82ccbdfa6a49a32c792304e7bb78e8b16471b29 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Thu, 10 Aug 2023 17:13:20 +0400 Subject: [PATCH 14/25] DOC: Improve the overview, functions signatures --- docs/markdown/Features-module.md | 307 ++++++++++++++++++------------- 1 file changed, 184 insertions(+), 123 deletions(-) diff --git a/docs/markdown/Features-module.md b/docs/markdown/Features-module.md index a0b07ad4b577..2d921fbd3a84 100644 --- a/docs/markdown/Features-module.md +++ b/docs/markdown/Features-module.md @@ -2,21 +2,24 @@ ## Overview -Dealing with numerous CPU features through C and C++ compilers is a challenging task, -especially when aiming to support massive amount of CPU features for various architectures and multiple compilers -Additionally, supporting both baseline features and additional features dispatched at runtime presents another dilemma. +Dealing with a numerous of CPU features within C and C++ compilers poses intricate challenges, +particularly when endeavoring to support an extensive set of CPU features across diverse +architectures and multiple compilers. Furthermore, the conundrum of accommodating +both fundamental features and supplementary features dispatched at runtime +further complicates matters. -Another issue that may arise is simplifying the implementations of generic interfaces while keeping the dirty work laid -on the build system rather than using nested namespaces or recursive sources, relying on pragma or compiler targets attributes -on top of complicated precompiled macros or meta templates, which can make debugging and maintenance difficult. +In addition, the task of streamlining generic interface implementations often leads to the necessity +of intricate solutions, which can obscure code readability and hinder debugging and maintenance. +Nested namespaces, recursive sources, complex precompiled macros, or intricate meta templates +are commonly employed but can result in convoluted code structures. -While this module doesn't force you to follow a specific approach, it instead paves the way to count on a -practical multi-targets solution that can make managing CPU features easier and more reliable. +Enter the proposed module, which offers a simple pragmatic multi-target solution to +facilitate the management of CPU features with heightened ease and reliability. -In a nutshell, this module helps you deliver the following concept: +In essence, this module introduces the following core principle: ```C -// Brings the headers files of enabled CPU features +// Include header files for enabled CPU features #ifdef HAVE_SSE #include #endif @@ -32,20 +35,10 @@ In a nutshell, this module helps you deliver the following concept: #ifdef HAVE_SSE41 #include #endif -#ifdef HAVE_POPCNT - #ifdef _MSC_VER - #include - #else - #include - #endif -#endif -#ifdef HAVE_AVX - #include -#endif - #ifdef HAVE_NEON #include #endif +// ... (similar blocks for other features) // MTARGETS_CURRENT defined as compiler argument via `features.multi_targets()` #ifdef MTARGETS_CURRENT @@ -55,19 +48,13 @@ In a nutshell, this module helps you deliver the following concept: #define TARGET_SFX(X) X #endif +// Core function utilizing feature-specific implementations void TARGET_SFX(my_kernal)(const float *src, float *dst) { -#ifdef HAVE_AVX512F - // defeintions for implied features alawys present - // no matter the compiler is - #ifndef HAVE_AVX2 - #error "Alawys defined" - #endif -#elif defined(HAVE_AVX2) && defined(HAVE_FMA3) - #ifndef HAVE_AVX - #error "Alawys defined" - #endif -#elif defined(HAVE_SSE41) +// Feature-based branching +#if defined(HAVE_SSE41) + // Definitions for implied features always present, + // regardless of the compiler used. #ifndef HAVE_SSSE3 #error "Alawys defined" #endif @@ -93,87 +80,107 @@ void TARGET_SFX(my_kernal)(const float *src, float *dst) } ``` -From the above code we can deduce the following: -- The code is written on top features based definitions rather than counting clusters or - features groups which gives the code more readability and flexibility. +Key Takeaways from the Code: -- Avoid using compiler built-in defeintions no matters the enabled arguments allows you - to easily manage the enabled/disabled features and to deal with any kind of compiler or features. - Since compilers like MSVC for example doesn't provides defeintions for all CPU features. +- The code employs feature-centric definitions, enhancing readability + and flexibility while sidestepping the need for feature grouping. -- The code is not aware of how its going to be build it, that gives the code a great prodiblity to - manage the generated objects which allow raising the baseline features at any time - or reduce and increase the additional dispatched features without changing the code. +- Notably, compiler-built definitions are circumvented, thereby affording + seamless management of enabled/disabled features and accommodating diverse compilers and feature sets. + Notably, this accommodates compilers like MSVC, which might lack definitions for specific CPU features. -- Allow building a single source multiple of times simplifying the implementations - of generic interfaces. +- The code remains agnostic about its build process, granting it remarkable versatility + in managing generated objects. This empowers the ability to elevate baseline features at will, + adjust additional dispatched features, and effect changes without necessitating code modifications. +- The architecture permits the construction of a singular source, which can be compiled multiple times. + This strategic approach simplifies the implementation of generic interfaces, streamlining the development process. -## Usage +## Usage To use this module, just do: **`features = import('features')`**. The following functions will then be available as methods on the object with the name `features`. You can, of course, replace the name `features` with anything else. ### features.new() + +**Signature** ```meson -features.new(string, int, - implies: FeatureOject | FeatureObject[] = [], - group: string | string[] = [], - detect: string | {} | (string | {})[] = [], - args: string | {} | (string | {})[] = [], - test_code: string | File = '', - extra_tests: {string: string | file} = {}, - disable: string = '' - ) -> FeatureObject +FeatureObject features.new( + # Positional arguments: + str, + int, + # Keyword arguments: + implies : FeatureOject | list[FeatureObject] = [], + group : str | list[str] = [], + args : str | dict[str] | list[str | dict[str]] = [], + detect : str | dict[str] | list[str | dict[str]] = [], + test_code : str | File = '', + extra_tests : dict[str | file] = {}, + disable : str = '' +) ``` -This function plays a crucial role in the Features Module as it creates -a new `FeatureObject` instance that is essential for the functioning of -other methods within the module. - -It takes two required positional arguments. The first one is the name of the feature, -and the second is the interest level of the feature, which is used by -the sort operation and priority of conflicting arguments. -The rest of the kwargs arguments are explained as follows: - -* `implies` **FeatureOject | FeatureObject[] = []**: One or an array of features objects - representing predecessor features. - -* `group` **string | string[] = []**: Optional one or an array of extra features names - to be added as extra definitions that can passed to source. - -* `args` **string | {} | (string | {})[] = []**: Optional one or an array of compiler - arguments that are required to be enabled for this feature. - Each argument can be a string or a dictionary that holds four items that allow dealing - with the conflicts of the arguments of implied features: - - `val` **string**: string, the compiler argument. - - `match` **string | empty**: regex to match the arguments of implied features - that need to be filtered or erased. - - `mfilter` **string | empty**: regex to find certain strings from the matched arguments - to be combined with `val`. If the value of `mfilter` is empty or undefined, - any matches triggered by the value of `match` will not be combined with `val`. - - `mjoin` **string | empty**: a separator used to join all the filtered arguments. - If it's empty or undefined, the filtered arguments will be joined without a separator. - -* `detect` **string | {} | (string | {})[] = []**: Optional one or an array of features names - that required to be detect on runtime. If no features sepecfied then the values of `group` - will be used if its provides otherwise the name of the feature will be used instead. - Similar to `args`, each feature name can be a string or a dictionary that holds four items - that allow dealing with the conflicts of the of implied features names. - See `features.multi_targets()` or `features.test()` for more clearfications. - -* `test_code` **string | File = ''**: Optional C/C++ code or the path to the source - that needs to be tested against the compiler to consider this feature is supported. - -* `extra_tests` **{string: string | file} = {}**: Optional dictionary holds extra tests where - the key represents the test name, which is also added as a compiler definition if the test succeeded, - and the value is C/C++ code or the path to the source that needs to be tested against the compiler. - -* `disable` **string = ''**: Optional string to consider this feature disabled. - -Returns a new instance of `FeatureObject`. +The `features.new()` function plays a pivotal role within the Features Module. +It creates a fresh mutable instance of FeatureObject, an essential component +for facilitating the functionality of other methods within the module. + +This function requires two positional arguments. The first argument pertains to the feature's name, +while the second one involves the interest level of the feature. This interest level governs sorting +operations and aids in resolving conflicts among arguments with differing priorities. + +Additional keyword arguments are elaborated as follows: + +* `implies` **FeatureOject | list[FeatureObject]**: + An optional single feature object or an array of feature objects + representing predecessor features. It is noteworthy that two features can imply each other. + Such mutual implication can prove beneficial in addressing compatibility concerns with compilers or hardware. + For instance, while some compilers might require enabling both `AVX2` and `FMA3` simultaneously, + others may permit independent activation. + +* `group` **str | list[str]**: + An optional single feature name or an array of additional feature names. These names are appended as + supplementary definitions that can be passed to the source. + +* `args` **str | dict[str] | list[str | dict[str]] = []**: + An optional single compiler argument or an array of compiler arguments that must be enabled for + the corresponding feature. Each argument can be a string or a dictionary containing four elements. + These elements handle conflicts arising from arguments of implied features or when concatenating two features: + - `val` **str**: + The compiler argument. + - `match` **str | empty**: + A regular expression to match arguments of implied features that necessitate filtering or removal. + - `mfilter` **str | empty**: + A regular expression to identify specific strings from matched arguments. + These strings are combined with `val`. If `mfilter` is empty or undefined, + matched arguments from `match` will not be combined with `val`. + - `mjoin` **str | empty**: + A separator used to join all filtered arguments. + If undefined or empty, filtered arguments are joined without a separator. + +* `detect` **str | dict[str] | list[str | dict[str]] = [] = []**: + An optional single feature name or an array of feature names to be detected at runtime. + If no feature names are specified, the values from the `group` will be used. + If the `group` doesn't provide values, the feature's name is employed instead. + Similar to args, each feature name can be a string or a dictionary with four + elements to manage conflicts of implied feature names. + Refer to `features.multi_targets()` or `features.test()` for further clarity. + +* `test_code` **str | File = ''**: + An optional block of C/C++ code or the path to a source file for testing against the compiler. + Successful compilation indicates feature support. + +* `extra_tests` **dict[str | file] = {}**: + An optional dictionary containing extra tests. The keys represent test names, + and the associated values are C/C++ code or paths to source files. + Successful tests lead to compiler definitions based on the test names. + +* `disable` **str = ''**: + An optional string to mark a feature as disabled. + If the string is empty or undefined, the feature is considered not disabled. + +The function returns a new instance of `FeatureObject`. Example: @@ -211,49 +218,103 @@ ASIMDFHM = features.new( args: {'val': '-march=armv8.2-a+fp16fml', 'match': '-march=.*', 'mfilter': '\+.*'} ) ``` - ### features.test() +**Signature** ```meson -features.test(FeatureObject..., - anyfet: bool = false, - force_args: string | string[] | empty = empty, - compiler: Compiler | empty = empty, - cached: bool = true, - ) -> {} +[bool, Dict] features.test( + # Positional arguments: + FeatureObject... + # Keyword arguments: + anyfet : bool = false, + force_args : str | list[str] | empty = empty, + compiler : Compiler | empty = empty, + cached : bool = true +) ``` -Test a one or set of features against the compiler and returns a dictionary -contains all required information that needed for building a source that -requires these features. +Test one or a list of features against the compiler and retrieve a dictionary containing +essential information required for compiling a source that depends on these features. + +A feature is deemed supported if it fulfills the following criteria: + + - It is not marked as disabled. + - The compiler accommodates the features arguments. + - Successful compilation of the designated test file or code. + - The implied features are supported by the compiler, aligning with + the criteria mentioned above. + +This function requires at least one feature object as positional argument, +and additional keyword arguments are elaborated as follows: + +- `anyfet` **bool = false**: + If set to true, the returned dictionary will encompass information regarding + the maximum features available that are supported by the compiler. + This extends beyond the specified features and includes their implied features. +- `force_args` **str | list[str] | empty = empty**: + An optional single compiler argument or an array of compiler arguments to be + employed instead of the designated features' arguments when testing the + test code against the compiler. This can be useful for detecting host features. +- `compiler` **Compiler | empty = empty**: + A Compiler object to be tested against. If not defined, + the function will default to the standard C or C++ compiler. +- `cached`: **bool = true**: + Enable or disable the cache. By default, the cache is enabled, + enhancing efficiency by storing previous results. + +This function returns a list of two items. The first item is a boolean value, +set to true if the compiler supports the feature and false if it does not. +The second item is a dictionary that encapsulates the test results, outlined as follows: + +**structure** +```meson +{ + 'target_name' : str, + 'prevalent_features' : list[str], + 'features' : list[str], + 'args' : list[str], + 'detect' : list[str], + 'defines' : list[str], + 'undefines' : list[str], + 'is_supported' : bool, + 'is_disabled' : bool, + 'fail_reason' : str +} +``` ### features.multi_targets() +**Signature** ```meson -features.multi_targets(string, ( - str | File | CustomTarget | CustomTargetIndex | - GeneratedList | StructuredSources | ExtractedObjects | - BuildTarget - )..., - dispatch: (FeatureObject | FeatureObject[])[] = [], - baseline: empty | FeatureObject[] = empty, - prefix: string = '', - compiler: empty | compiler = empty, - cached: bool = True - ) [{}[], StaticLibrary[]] +TargetsObject features.multi_targets( + # Positional arguments: + str, + ( + str | File | CustomTarget | CustomTargetIndex | + GeneratedList | StructuredSources | ExtractedObjects | + BuildTarget + )..., + # Keyword arguments: + dispatch : FeatureObject | list[FeatureObject] = [], + baseline : empty | list[FeatureObject] = empty, + prefix : str = '', + keep_sort : bool = false, + compiler : empty | compiler = empty, + cached : bool = True, + **known_stlib_kwargs +) ``` - ### features.sort() +**Signature** ```meson -features.sort(FeatureObject..., reverse: bool = false) : FeatureObject[] ``` ### features.implicit() +**Signature** ```meson -features.implicit(FeatureObject...) : FeatureObject[] ``` ### features.implicit_c() +**Signature** ```meson -features.implicit_c(FeatureObject...) : FeatureObject[] ``` From 55fb7df42b725164dadbbb5fa23ed94020346bb4 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Sun, 10 Sep 2023 11:54:28 +0400 Subject: [PATCH 15/25] BUG: Fix features tests on cross-compile builds --- mesonbuild/modules/features/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/modules/features/utils.py b/mesonbuild/modules/features/utils.py index 74cb45d8648b..88eb19d82d92 100644 --- a/mesonbuild/modules/features/utils.py +++ b/mesonbuild/modules/features/utils.py @@ -9,7 +9,7 @@ from .. import ModuleState def get_compiler(state: 'ModuleState') -> 'Compiler': - for_machine = MachineChoice.BUILD + for_machine = MachineChoice.HOST clist = state.environment.coredata.compilers[for_machine] for cstr in ('c', 'cpp'): try: From ae3b57f9fe141f481fcef514d28e29e90c237a02 Mon Sep 17 00:00:00 2001 From: Fabian Vogt Date: Thu, 28 Mar 2024 13:15:54 +0100 Subject: [PATCH 16/25] feature module: Fix handling of multiple conflicts per attribute - Attributes without match were never actually added to the list - Only the last conflict actually had an effect, earlier results were discarded --- mesonbuild/modules/features/module.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mesonbuild/modules/features/module.py b/mesonbuild/modules/features/module.py index 0be6af06c8b7..a6f357b3f65b 100644 --- a/mesonbuild/modules/features/module.py +++ b/mesonbuild/modules/features/module.py @@ -371,15 +371,12 @@ def fail_result(fail_reason: str, is_disabled: bool = False values: List[ConflictAttr] = getattr(fet, attr) accumulate_values = test_result[attr] # type: ignore for conflict in values: - if not conflict.match: - accumulate_values.append(conflict.val) - continue conflict_vals: List[str] = [] # select the acc items based on the match new_acc: List[str] = [] for acc in accumulate_values: # not affected by the match so we keep it - if not conflict.match.match(acc): + if not (conflict.match and conflict.match.match(acc)): new_acc.append(acc) continue # no filter so we totaly escape it @@ -396,7 +393,7 @@ def fail_result(fail_reason: str, is_disabled: bool = False continue conflict_vals.append(conflict.mjoin.join(filter_val)) new_acc.append(conflict.val + conflict.mjoin.join(conflict_vals)) - test_result[attr] = new_acc # type: ignore + accumulate_values = test_result[attr] = new_acc # type: ignore test_args = compiler.has_multi_arguments args = test_result['args'] From 6bf7fefb72f4c4b7945039e07dbe428c54900b2c Mon Sep 17 00:00:00 2001 From: Sayed Awad Date: Mon, 8 Jun 2026 19:34:23 +0300 Subject: [PATCH 17/25] Avoid using symlink, merge all the module tests into a single project --- test cases/features/1 baseline/init_features | 1 - .../features/2 multi_targets/init_features | 1 - .../{1 baseline => test/baseline}/baseline.c | 0 .../{1 baseline => test/baseline}/meson.build | 3 --- .../checks => test/features}/cpu_asimd.c | 0 .../checks => test/features}/cpu_neon.c | 0 .../checks => test/features}/cpu_neon_fp16.c | 0 .../checks => test/features}/cpu_neon_vfpv4.c | 0 .../checks => test/features}/cpu_sse.c | 0 .../checks => test/features}/cpu_sse2.c | 0 .../checks => test/features}/cpu_sse3.c | 0 .../checks => test/features}/cpu_sse41.c | 0 .../checks => test/features}/cpu_ssse3.c | 0 .../features}/meson.build | 20 +++++++++---------- test cases/features/test/meson.build | 4 ++++ .../multi_targets}/dispatch.h | 0 .../multi_targets}/dispatch1.c | 0 .../multi_targets}/dispatch2.c | 0 .../multi_targets}/dispatch3.c | 0 .../multi_targets}/main.c | 0 .../multi_targets}/meson.build | 3 --- 21 files changed, 13 insertions(+), 19 deletions(-) delete mode 120000 test cases/features/1 baseline/init_features delete mode 120000 test cases/features/2 multi_targets/init_features rename test cases/features/{1 baseline => test/baseline}/baseline.c (100%) rename test cases/features/{1 baseline => test/baseline}/meson.build (86%) rename test cases/features/{init_features/checks => test/features}/cpu_asimd.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_neon.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_neon_fp16.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_neon_vfpv4.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_sse.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_sse2.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_sse3.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_sse41.c (100%) rename test cases/features/{init_features/checks => test/features}/cpu_ssse3.c (100%) rename test cases/features/{init_features => test/features}/meson.build (80%) create mode 100644 test cases/features/test/meson.build rename test cases/features/{2 multi_targets => test/multi_targets}/dispatch.h (100%) rename test cases/features/{2 multi_targets => test/multi_targets}/dispatch1.c (100%) rename test cases/features/{2 multi_targets => test/multi_targets}/dispatch2.c (100%) rename test cases/features/{2 multi_targets => test/multi_targets}/dispatch3.c (100%) rename test cases/features/{2 multi_targets => test/multi_targets}/main.c (100%) rename test cases/features/{2 multi_targets => test/multi_targets}/meson.build (90%) diff --git a/test cases/features/1 baseline/init_features b/test cases/features/1 baseline/init_features deleted file mode 120000 index d52da150db76..000000000000 --- a/test cases/features/1 baseline/init_features +++ /dev/null @@ -1 +0,0 @@ -../init_features \ No newline at end of file diff --git a/test cases/features/2 multi_targets/init_features b/test cases/features/2 multi_targets/init_features deleted file mode 120000 index d52da150db76..000000000000 --- a/test cases/features/2 multi_targets/init_features +++ /dev/null @@ -1 +0,0 @@ -../init_features \ No newline at end of file diff --git a/test cases/features/1 baseline/baseline.c b/test cases/features/test/baseline/baseline.c similarity index 100% rename from test cases/features/1 baseline/baseline.c rename to test cases/features/test/baseline/baseline.c diff --git a/test cases/features/1 baseline/meson.build b/test cases/features/test/baseline/meson.build similarity index 86% rename from test cases/features/1 baseline/meson.build rename to test cases/features/test/baseline/meson.build index 71c4299cf3d4..277b3e636e52 100644 --- a/test cases/features/1 baseline/meson.build +++ b/test cases/features/test/baseline/meson.build @@ -1,6 +1,3 @@ -project('baseline', 'c') -subdir('init_features') - baseline = mod_features.test(SSE3, NEON, anyfet: true) message(baseline) baseline_args = baseline[1]['args'] diff --git a/test cases/features/init_features/checks/cpu_asimd.c b/test cases/features/test/features/cpu_asimd.c similarity index 100% rename from test cases/features/init_features/checks/cpu_asimd.c rename to test cases/features/test/features/cpu_asimd.c diff --git a/test cases/features/init_features/checks/cpu_neon.c b/test cases/features/test/features/cpu_neon.c similarity index 100% rename from test cases/features/init_features/checks/cpu_neon.c rename to test cases/features/test/features/cpu_neon.c diff --git a/test cases/features/init_features/checks/cpu_neon_fp16.c b/test cases/features/test/features/cpu_neon_fp16.c similarity index 100% rename from test cases/features/init_features/checks/cpu_neon_fp16.c rename to test cases/features/test/features/cpu_neon_fp16.c diff --git a/test cases/features/init_features/checks/cpu_neon_vfpv4.c b/test cases/features/test/features/cpu_neon_vfpv4.c similarity index 100% rename from test cases/features/init_features/checks/cpu_neon_vfpv4.c rename to test cases/features/test/features/cpu_neon_vfpv4.c diff --git a/test cases/features/init_features/checks/cpu_sse.c b/test cases/features/test/features/cpu_sse.c similarity index 100% rename from test cases/features/init_features/checks/cpu_sse.c rename to test cases/features/test/features/cpu_sse.c diff --git a/test cases/features/init_features/checks/cpu_sse2.c b/test cases/features/test/features/cpu_sse2.c similarity index 100% rename from test cases/features/init_features/checks/cpu_sse2.c rename to test cases/features/test/features/cpu_sse2.c diff --git a/test cases/features/init_features/checks/cpu_sse3.c b/test cases/features/test/features/cpu_sse3.c similarity index 100% rename from test cases/features/init_features/checks/cpu_sse3.c rename to test cases/features/test/features/cpu_sse3.c diff --git a/test cases/features/init_features/checks/cpu_sse41.c b/test cases/features/test/features/cpu_sse41.c similarity index 100% rename from test cases/features/init_features/checks/cpu_sse41.c rename to test cases/features/test/features/cpu_sse41.c diff --git a/test cases/features/init_features/checks/cpu_ssse3.c b/test cases/features/test/features/cpu_ssse3.c similarity index 100% rename from test cases/features/init_features/checks/cpu_ssse3.c rename to test cases/features/test/features/cpu_ssse3.c diff --git a/test cases/features/init_features/meson.build b/test cases/features/test/features/meson.build similarity index 80% rename from test cases/features/init_features/meson.build rename to test cases/features/test/features/meson.build index c0b4f13fa862..195d5aae1d93 100644 --- a/test cases/features/init_features/meson.build +++ b/test cases/features/test/features/meson.build @@ -1,18 +1,16 @@ -#project('test-features', 'c') mod_features = import('features') cpu_family = host_machine.cpu_family() compiler_id = meson.get_compiler('c').get_id() -source_root = meson.project_source_root() + '/../init_features/' # Basic X86 Features # ------------------ SSE = mod_features.new( 'SSE', 1, args: '-msse', - test_code: files(source_root + 'checks/cpu_sse.c')[0] + test_code: files('cpu_sse.c')[0] ) SSE2 = mod_features.new( 'SSE2', 2, implies: SSE, args: '-msse2', - test_code: files(source_root + 'checks/cpu_sse2.c')[0] + test_code: files('cpu_sse2.c')[0] ) # enabling SSE without SSE2 is useless also # it's non-optional for x86_64 @@ -20,17 +18,17 @@ SSE.update(implies: SSE2) SSE3 = mod_features.new( 'SSE3', 3, implies: SSE2, args: '-msse3', - test_code: files(source_root + 'checks/cpu_sse3.c')[0] + test_code: files('cpu_sse3.c')[0] ) SSSE3 = mod_features.new( 'SSSE3', 4, implies: SSE3, args: '-mssse3', - test_code: files(source_root + 'checks/cpu_ssse3.c')[0] + test_code: files('cpu_ssse3.c')[0] ) SSE41 = mod_features.new( 'SSE41', 5, implies: SSSE3, args: '-msse4.1', - test_code: files(source_root + 'checks/cpu_sse41.c')[0] + test_code: files('cpu_sse41.c')[0] ) if cpu_family not in ['x86', 'x86_64'] # should disable any prevalent features @@ -62,21 +60,21 @@ endif # ------------------ NEON = mod_features.new( 'NEON', 200, - test_code: files(source_root + 'checks/cpu_neon.c')[0] + test_code: files('cpu_neon.c')[0] ) NEON_FP16 = mod_features.new( 'NEON_FP16', 201, implies: NEON, - test_code: files(source_root + 'checks/cpu_neon_fp16.c')[0] + test_code: files('cpu_neon_fp16.c')[0] ) # FMA NEON_VFPV4 = mod_features.new( 'NEON_VFPV4', 202, implies: NEON_FP16, - test_code: files(source_root + 'checks/cpu_neon_vfpv4.c')[0] + test_code: files('cpu_neon_vfpv4.c')[0] ) # Advanced SIMD ASIMD = mod_features.new( 'ASIMD', 203, implies: NEON_VFPV4, detect: {'val': 'ASIMD', 'match': 'NEON.*'}, - test_code: files(source_root + 'checks/cpu_asimd.c')[0] + test_code: files('cpu_asimd.c')[0] ) if cpu_family == 'aarch64' # hardware baseline, they can't be enabled independently diff --git a/test cases/features/test/meson.build b/test cases/features/test/meson.build new file mode 100644 index 000000000000..474cdf80f9ac --- /dev/null +++ b/test cases/features/test/meson.build @@ -0,0 +1,4 @@ +project('test-features', 'c') +subdir('features') +subdir('baseline') +subdir('multi_targets') diff --git a/test cases/features/2 multi_targets/dispatch.h b/test cases/features/test/multi_targets/dispatch.h similarity index 100% rename from test cases/features/2 multi_targets/dispatch.h rename to test cases/features/test/multi_targets/dispatch.h diff --git a/test cases/features/2 multi_targets/dispatch1.c b/test cases/features/test/multi_targets/dispatch1.c similarity index 100% rename from test cases/features/2 multi_targets/dispatch1.c rename to test cases/features/test/multi_targets/dispatch1.c diff --git a/test cases/features/2 multi_targets/dispatch2.c b/test cases/features/test/multi_targets/dispatch2.c similarity index 100% rename from test cases/features/2 multi_targets/dispatch2.c rename to test cases/features/test/multi_targets/dispatch2.c diff --git a/test cases/features/2 multi_targets/dispatch3.c b/test cases/features/test/multi_targets/dispatch3.c similarity index 100% rename from test cases/features/2 multi_targets/dispatch3.c rename to test cases/features/test/multi_targets/dispatch3.c diff --git a/test cases/features/2 multi_targets/main.c b/test cases/features/test/multi_targets/main.c similarity index 100% rename from test cases/features/2 multi_targets/main.c rename to test cases/features/test/multi_targets/main.c diff --git a/test cases/features/2 multi_targets/meson.build b/test cases/features/test/multi_targets/meson.build similarity index 90% rename from test cases/features/2 multi_targets/meson.build rename to test cases/features/test/multi_targets/meson.build index ffceda098f1a..b310c8458ba1 100644 --- a/test cases/features/2 multi_targets/meson.build +++ b/test cases/features/test/multi_targets/meson.build @@ -1,6 +1,3 @@ -project('multi_targets', 'c') -subdir('init_features') - multi_targets = mod_features.multi_targets( 'dispatch1.conf.h', 'dispatch1.c', dispatch: [SSSE3, ASIMD], From 4b7735bfec34dca4830af1be345878b67ea4e260 Mon Sep 17 00:00:00 2001 From: Sayed Awad Date: Mon, 8 Jun 2026 19:36:24 +0300 Subject: [PATCH 18/25] fix linting problems & make the module compatible with latest version --- mesonbuild/modules/features/feature.py | 14 +++--- mesonbuild/modules/features/module.py | 37 ++++++++------- mesonbuild/modules/features/utils.py | 5 +- unittests/featurestests.py | 66 ++++++++++++++++++-------- 4 files changed, 75 insertions(+), 47 deletions(-) diff --git a/mesonbuild/modules/features/feature.py b/mesonbuild/modules/features/feature.py index 7e0f621e543f..1bc603773301 100644 --- a/mesonbuild/modules/features/feature.py +++ b/mesonbuild/modules/features/feature.py @@ -3,7 +3,7 @@ import re from typing import ( Dict, Set, Tuple, List, Callable, Optional, - Union, Any, Iterable, cast, TYPE_CHECKING + Union, Any, Iterable, TYPE_CHECKING ) from dataclasses import dataclass, field from ...mesonlib import File, MesonException @@ -18,7 +18,6 @@ from typing import TypedDict from typing_extensions import NotRequired from ...interpreterbase import TYPE_var, TYPE_kwargs - from ...compilers import Compiler from .. import ModuleState @dataclass(unsafe_hash=True, order=True) @@ -80,7 +79,7 @@ def __init__(self, func_name: str, opt_name: str, default: Any = None): ) @staticmethod - def convert(func_name:str, opt_name: str, values: 'IMPLIED_ATTR', + def convert(func_name: str, opt_name: str, values: 'IMPLIED_ATTR', ) -> Union[None, List[ConflictAttr]]: if values is None: return None @@ -131,6 +130,7 @@ def convert(func_name:str, opt_name: str, values: 'IMPLIED_ATTR', Union[str, Dict[str, str]] ] ] + class FeatureKwArgs(TypedDict): #implies: Optional[List['FeatureObject']] implies: NotRequired[List[Any]] @@ -163,7 +163,8 @@ def __init__(self, state: 'ModuleState', super().__init__() @typed_pos_args('features.new', str, int) - @typed_kwargs('features.new', + @typed_kwargs( + 'features.new', KwargInfo( 'implies', (FeatureObject, ContainerTypeInfo(list, FeatureObject)), @@ -209,7 +210,8 @@ def init_attrs(state: 'ModuleState', def update_method(self, state: 'ModuleState', args: List['TYPE_var'], kwargs: 'TYPE_kwargs') -> 'FeatureObject': @noPosargs - @typed_kwargs('features.FeatureObject.update', + @typed_kwargs( + 'features.FeatureObject.update', KwargInfo('name', (NoneType, str)), KwargInfo('interest', (NoneType, int)), KwargInfo( @@ -306,7 +308,7 @@ def sort_cb(k: Union[FeatureObject, Iterable[FeatureObject]]) -> int: # FIXME: that's not a safe way to increase the rank for # multi features this why this function isn't considerd # accurate. - rank += len(prevalent_features) -1 + rank += len(prevalent_features) - 1 return rank return sorted(features, reverse=reverse, key=sort_cb) diff --git a/mesonbuild/modules/features/module.py b/mesonbuild/modules/features/module.py index a6f357b3f65b..23c01c0dbd70 100644 --- a/mesonbuild/modules/features/module.py +++ b/mesonbuild/modules/features/module.py @@ -10,7 +10,7 @@ from ...interpreter.type_checking import NoneType from ...interpreterbase.decorators import ( noKwargs, KwargInfo, typed_kwargs, typed_pos_args, - ContainerTypeInfo, permittedKwargs + ContainerTypeInfo ) from .. import ModuleInfo, NewExtensionModule, ModuleObject from .feature import FeatureObject, ConflictAttr @@ -20,7 +20,6 @@ from typing import TypedDict from ...interpreterbase import TYPE_var, TYPE_kwargs from .. import ModuleState - from .feature import FeatureKwArgs class TestKwArgs(TypedDict): compiler: Optional[Compiler] @@ -104,11 +103,12 @@ def add_target(self, features: Union[FeatureObject, List[FeatureObject]], else tuple(sorted(features)) ) targets: List[build.StaticLibrary] = self._targets.setdefault( - tfeatures, cast(List[build.StaticLibrary], [])) # type: ignore + tfeatures, []) targets.append(target) class Module(NewExtensionModule): INFO = ModuleInfo('features', '0.1.0') + def __init__(self) -> None: super().__init__() self.methods.update({ @@ -142,7 +142,8 @@ def _set_cache(self, state: 'ModuleState', key: str, self._cache_dict(state)[key] = val @typed_pos_args('features.test', varargs=FeatureObject, min_varargs=1) - @typed_kwargs('features.test', + @typed_kwargs( + 'features.test', KwargInfo('compiler', (NoneType, Compiler)), KwargInfo('anyfet', bool, default = False), KwargInfo('cached', bool, default = True), @@ -255,7 +256,7 @@ def test_any(self, state: 'ModuleState', features: Set[FeatureObject], features_any = set() for fet in all_features: _, test_any_result = self.cached_test( - state, features={fet,}, + state, features={fet, }, compiler=compiler, cached=cached, anyfet=False, @@ -293,7 +294,7 @@ def test(self, state: 'ModuleState', features: Set[FeatureObject], # Set the highest interested feature prevalent_features = sorted(features)[-1:] - prevalent_names = [fet.name for fet in prevalent_features] + prevalent_names = [fet.name for fet in prevalent_features] # prepare the result dict test_result: 'TestResultKwArgs' = { 'target_name': '__'.join(prevalent_names), @@ -307,6 +308,7 @@ def test(self, state: 'ModuleState', features: Set[FeatureObject], 'is_disabled': False, 'fail_reason': '', } + def fail_result(fail_reason: str, is_disabled: bool = False ) -> 'TestResultKwArgs': test_result.update({ @@ -337,7 +339,7 @@ def fail_result(fail_reason: str, is_disabled: bool = False predecessor_features = implied_features.difference(_caller) for fet in sorted(predecessor_features): _, pred_result = self.cached_test( - state, features={fet,}, + state, features={fet, }, compiler=compiler, cached=cached, anyfet=False, @@ -398,7 +400,7 @@ def fail_result(fail_reason: str, is_disabled: bool = False test_args = compiler.has_multi_arguments args = test_result['args'] if args: - supported_args, test_cached = test_args(args, state.environment) + supported_args, test_cached = test_args(args) if not supported_args: return fail_result( f'Arguments "{", ".join(args)}" are not supported' @@ -423,15 +425,13 @@ def fail_result(fail_reason: str, is_disabled: bool = False test_result[k].append(extra_name) # type: ignore return test_result - @permittedKwargs(build.known_stlib_kwargs | { - 'dispatch', 'baseline', 'prefix', 'cached', 'keep_sort' - }) @typed_pos_args('features.multi_targets', str, min_varargs=1, varargs=( str, File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList, build.StructuredSources, build.ExtractedObjects, build.BuildTarget )) - @typed_kwargs('features.multi_targets', + @typed_kwargs( + 'features.multi_targets', KwargInfo( 'dispatch', ( ContainerTypeInfo(list, (FeatureObject, list)), @@ -451,8 +451,8 @@ def fail_result(fail_reason: str, is_disabled: bool = False allow_unknown=True ) def multi_targets_method(self, state: 'ModuleState', - args: Tuple[str], kwargs: 'TYPE_kwargs' - ) -> TargetsObject: + args: Tuple[str], kwargs: 'TYPE_kwargs' + ) -> TargetsObject: config_name = args[0] sources = args[1] # type: ignore dispatch: List[Union[FeatureObject, List[FeatureObject]]] = ( @@ -467,7 +467,7 @@ def multi_targets_method(self, state: 'ModuleState', if not compiler: compiler = get_compiler(state) - baseline_features : Set[FeatureObject] = set() + baseline_features: Set[FeatureObject] = set() has_baseline = baseline is not None if has_baseline: baseline_features = FeatureObject.get_implicit_combine_multi(baseline) @@ -488,7 +488,7 @@ def multi_targets_method(self, state: 'ModuleState', ]] = [] for d in dispatch: if isinstance(d, FeatureObject): - target = {d,} + target = {d, } is_base_part = d in baseline_features else: target = set(d) @@ -647,7 +647,7 @@ def gen_config(self, state: 'ModuleState', config_name: str, c_detect = '1' dispatch_calls.append( f'{prefix}_MTARGETS_EXPAND(' - f'EXEC_CB({c_detect}, {test["target_name"]}, __VA_ARGS__)' + f'EXEC_CB({c_detect}, {test["target_name"]}, __VA_ARGS__)' ')' ) @@ -683,7 +683,8 @@ def gen_config(self, state: 'ModuleState', config_name: str, return config_path @typed_pos_args('features.sort', varargs=FeatureObject, min_varargs=1) - @typed_kwargs('features.sort', + @typed_kwargs( + 'features.sort', KwargInfo('reverse', bool, default = False), ) def sort_method(self, state: 'ModuleState', diff --git a/mesonbuild/modules/features/utils.py b/mesonbuild/modules/features/utils.py index 88eb19d82d92..84a2a46179a2 100644 --- a/mesonbuild/modules/features/utils.py +++ b/mesonbuild/modules/features/utils.py @@ -2,9 +2,9 @@ import hashlib from typing import Tuple, List, Union, Any, TYPE_CHECKING from ...mesonlib import MesonException, MachineChoice +from ...compilers.compilers import CompileCheckMode, Compiler if TYPE_CHECKING: - from ...compilers import Compiler from ...mesonlib import File from .. import ModuleState @@ -27,13 +27,12 @@ def test_code(state: 'ModuleState', compiler: 'Compiler', ) -> Tuple[bool, bool, str]: # TODO: Add option to treat warnings as errors with compiler.cached_compile( - code, state.environment.coredata, extra_args=args + code, extra_args=args, mode=CompileCheckMode.COMPILE ) as p: return p.cached, p.returncode == 0, p.stderr def generate_hash(*args: Any) -> str: hasher = hashlib.sha1() - test: List[bytes] = [] for a in args: hasher.update(bytes(str(a), encoding='utf-8')) return hasher.hexdigest() diff --git a/unittests/featurestests.py b/unittests/featurestests.py index a863a9fb93cc..cc09c277fd60 100644 --- a/unittests/featurestests.py +++ b/unittests/featurestests.py @@ -1,10 +1,12 @@ # Copyright (c) 2023, NumPy Developers. import re +import os import contextlib +from unittest import mock from mesonbuild.interpreter import Interpreter from mesonbuild.build import Build -from mesonbuild.mparser import FunctionNode, ArgumentNode, Token +from mesonbuild.mparser import IdNode, SymbolNode, Token, FunctionNode, ArgumentNode from mesonbuild.modules import ModuleState from mesonbuild.modules.features import Module from mesonbuild.compilers import Compiler, CompileResult @@ -12,7 +14,7 @@ from mesonbuild.envconfig import MachineInfo from .baseplatformtests import BasePlatformTests -from run_tests import get_convincing_fake_env_and_cc +from run_tests import FakeBuild, get_fake_env, get_convincing_fake_env_and_cc class FakeCompiler(Compiler): language = 'c' @@ -20,13 +22,8 @@ class FakeCompiler(Compiler): def __init__(self, trap_args = '', trap_code=''): super().__init__( ccache=[], exelist=[], version='0.0', + environment=get_fake_env(), for_machine=MachineChoice.HOST, - info=MachineInfo( - system='linux', cpu_family='x86_64', - cpu='xeon', endian='little', - kernel='linux', subsystem='numpy' - ), - is_cross=True ) self.trap_args = trap_args self.trap_code = trap_code @@ -40,7 +37,7 @@ def get_optimization_args(self, optimization_level: str) -> 'T.List[str]': def get_output_args(self, outputname: str) -> 'T.List[str]': return [] - def has_multi_arguments(self, args: 'T.List[str]', env: 'Environment') -> 'T.Tuple[bool, bool]': + def has_multi_arguments(self, args: 'T.List[str]') -> 'T.Tuple[bool, bool]': if self.trap_args: for a in args: if re.match(self.trap_args, a): @@ -54,7 +51,13 @@ def compile(self, code: 'mesonlib.FileOrString', *args, **kwargs rcode = -1 else: rcode = 0 - result = CompileResult(returncode=rcode) + result = CompileResult( + stdout='', + stderr='', + command=[], + returncode=rcode, + input_name='fake_input.c' + ) yield result @contextlib.contextmanager @@ -64,28 +67,51 @@ def cached_compile(self, code: 'mesonlib.FileOrString', *args, **kwargs rcode = -1 else: rcode = 0 - result = CompileResult(returncode=rcode) + result = CompileResult( + stdout='', + stderr='', + command=[], + returncode=rcode, + input_name='fake_input.c' + ) yield result + + def _sanity_check_source_code(self) -> str: + return 'public static int main() { return 0; }' + class FeaturesTests(BasePlatformTests): + + @mock.patch.dict(os.environ) + @mock.patch.object(Interpreter, 'load_root_meson_file', mock.Mock(return_value=None)) + @mock.patch.object(Interpreter, 'sanity_check_ast', mock.Mock(return_value=None)) + @mock.patch.object(Interpreter, 'parse_project', mock.Mock(return_value=None)) def setUp(self): super().setUp() env, cc = get_convincing_fake_env_and_cc( bdir=self.builddir, prefix=self.prefix) env.machines.target = env.machines.host + # Disable unit test specific syntax + os.environ.pop('MESON_RUNNING_IN_PROJECT_TESTS', None) build = Build(env) - interp = Interpreter(build, mock=True) + interp = Interpreter(build) + # Mock out user_defined_options to bypass the coredata initialization crash + interp.user_defined_options = mock.Mock() + interp.user_defined_options.cmd_line_options = {} + project = interp.funcs['project'] filename = 'featurestests.py' + dummy_span = (0, 0) # Safe placeholder for bytespan: Tuple[int, int] + func_name_token = Token('id', filename, 0, 0, 0, dummy_span, 'FeaturesTests') + lpar_token = Token('lpar', filename, 0, 0, 13, dummy_span, '(') + arg_token = Token('string', filename, 0, 0, 14, dummy_span, '') + rpar_token = Token('rpar', filename, 0, 0, 15, dummy_span, ')') node = FunctionNode( - filename = filename, - lineno = 0, - colno = 0, - end_lineno = 0, - end_colno = 0, - func_name = 'FeaturesTests', - args = ArgumentNode(Token('string', filename, 0, 0, 0, None, '')) + IdNode(func_name_token), + SymbolNode(lpar_token), + ArgumentNode(arg_token), + SymbolNode(rpar_token) ) project(node, ['Test Module Features'], {'version': '0.1'}) self.cc = cc @@ -272,7 +298,7 @@ def test_conflict_args(self): ), ( [fet6], - 'args', {'val':'arch=', 'match': 'arg.*[0-9]|arch=.*', 'mfilter': '([0-9])|arch=(\w+)', 'mjoin': '*'}, + 'args', {'val':'arch=', 'match': 'arg.*[0-9]|arch=.*', 'mfilter': r'([0-9])|arch=(\w+)', 'mjoin': '*'}, ['arch=1*2*3*4*5*xx'], ), ( From 6b1f295b25cbbf93c9c61cd263a8db99e837f10e Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 10 Oct 2023 00:05:09 +0200 Subject: [PATCH 19/25] Implement BLAS/LAPACK API and dependencies: OpenBLAS, MKL, Accelerate --- docs/markdown/Dependencies.md | 157 ++++ mesonbuild/dependencies/__init__.py | 7 + mesonbuild/dependencies/blas_lapack.py | 753 ++++++++++++++++++ .../frameworks/38 blas-lapack/cblas_lapack.c | 47 ++ .../frameworks/38 blas-lapack/cblas_lapack.h | 113 +++ test cases/frameworks/38 blas-lapack/main.f90 | 25 + .../frameworks/38 blas-lapack/meson.build | 100 +++ .../frameworks/38 blas-lapack/test.json | 3 + 8 files changed, 1205 insertions(+) create mode 100644 mesonbuild/dependencies/blas_lapack.py create mode 100644 test cases/frameworks/38 blas-lapack/cblas_lapack.c create mode 100644 test cases/frameworks/38 blas-lapack/cblas_lapack.h create mode 100644 test cases/frameworks/38 blas-lapack/main.f90 create mode 100644 test cases/frameworks/38 blas-lapack/meson.build create mode 100644 test cases/frameworks/38 blas-lapack/test.json diff --git a/docs/markdown/Dependencies.md b/docs/markdown/Dependencies.md index eb72fd6efd7d..95ada01f7698 100644 --- a/docs/markdown/Dependencies.md +++ b/docs/markdown/Dependencies.md @@ -327,6 +327,163 @@ to what is provided by the C runtime libraries. `method` may be `auto`, `builtin` or `system`. +## BLAS and LAPACK + +*(numpy/meson#2)* + +Enables compiling and linking against BLAS and LAPACK libraries. BLAS and +LAPACK are generic APIs, which can be provided by a number of different +implementations. It is possible to request either any implementation that +provides the API, or a specific implementation like OpenBLAS or MKL. +Furthermore, a preferred order may be specified, as well as the bitness (32 or +64) of the interface and whether to use the C or Fortran APIs. + +Using the generic `'blas'` or `'lapack'` will try to find any matching +implementation. The search order over implementations is not guaranteed; Meson +aims to search them from higher to lower performance. + +```meson +dependency('blas', interface : 'lp64', cblas : true) +dependency('lapack', interface : 'ilp64', lapacke : false) +``` + +Keywords: +- `interface`: options are `lp64` for 32-bit LP64, and `ilp64` for 64-bit + ILP64. Default is `lp64`. +- `cblas`: `true` or `false`. Default is `true`. (TBD: is `auto` needed?) +- `lapacke`: `true` or `false`. Default is `false`. + +The search order can be controlled by explicitly specifying it: + +```meson +# Specify what implementations to look for, and in what order +blas_dep = dependency('accelerate', 'mkl', 'openblas', 'blis', 'atlas', 'netlib') +lapack_dep = dependency('accelerate', 'mkl', 'openblas', 'atlas', 'netlib') + +# You can also add the generic BLAS or LAPACK as a fallback, this may help +# portability +blas_dep = dependency('openblas', 'mkl', 'blis', 'netlib', 'blas') +lapack_dep = dependency('openblas', 'mkl', 'atlas', 'netlib') +``` + +Note that it is not necessary to specify both `'lapack'` and `'blas'` for the +same build target, because LAPACK itself depends on BLAS. (TBD: is this okay +for libflame?). + + +### Specific BLAS and LAPACK implementations + +#### OpenBLAS + +The `version` and `interface` keywords may be passed to request the use of a +specific version and interface, correspondingly: + +```meson +openblas_dep = dependency('openblas', + version : '>=0.3.21', + language: 'c', # can be c/cpp/fortran + modules: [ + 'interface: ilp64', # can be lp64 or ilp64 (or auto?) + 'symbol-suffix: 64_', # check/auto-detect? default to 64_ or no suffix? + 'cblas', + 'lapack', # OpenBLAS can be built without LAPACK support + ] +) + +# Query properties as needed: +has_cblas = openblas.get_variable('cblas') +is_ilp64 = openblas_dep.get_variable('interface') == 'ilp64' +blas_symbol_suffix = openblas_dep.get_variable('symbol-suffix') +``` + +If OpenBLAS is installed in a nonstandard location *with* pkg-config files, +you can set `PKG_CONFIG_PATH`. Alternatively, you can specify +`openblas_includedir` and `openblas_librarydir` in your native or cross machine +file, this works also if OpenBLAS is installed *without* pkg-config files: + +```ini +[properties] +openblas_includedir = '/path/to/include_dir' # should contain openblas_config.h +openblas_librarydir = '/path/to/library_dir' +``` + +Note that OpenBLAS can be built with either pthreads or OpenMP. Information on +this is not available through Meson. + +#### MKL + +```meson +mkl_dep = dependency('mkl', + version: '>=2021.1.0', + modules: [ + interface: 'lp64', # options are 'lp64' or 'ilp64' + threading: 'seq', # options are 'seq' or 'omp' + library: 'dynamic', # options are 'dynamic' or 'static' + ] +) +``` + +#### Netlib BLAS and LAPACK + +```meson +netlib_blas_dep = dependency('netlib-blas', version: '>=3.9.0') +netlib_lapack_dep = dependency('netlib-lapack', version: '>=3.9.0') +``` + +Note that this dependency will look for `libblas` or `liblapack`. No attempt is made +to enforce that they're the original Netlib reference libraries; if another library +is built with the same name, it's assumed that the APIs match. + +TODO: the Netlib BLAS library typically ships CBLAS as a separate library and a separate + cblas.pc file. + + +#### ArmPL + +```meson +armpl_dep = dependency('armpl', + version: '>=22.1', + modules: [ + interface: 'lp64', + threading: 'seq', + library: 'dynamic', + ] +) +``` + +The options for the `interface`, `threading` and `library` modules are the same +as for MKL. + + + +#### ATLAS + +TODO + +#### BLIS + +TODO + +### libflame + +TODO + +### Accelerate + +Supports the BLAS and LAPACK components of macOS Accelerate, also referred to +as the vecLib framework. ILP64 support is only available on macOS 13.3 and up. +From macOS 13.3, Accelerate ships with two different builds of 32-bit (LP64) +BLAS and LAPACK. Meson will default to the newer of those builds, by defining +`ACCELERATE_NEW_LAPACK`, unless `MACOS_DEPLOYMENT_TARGET` is set to a version +lower than 13.3. + +```meson +accelerate_dep = dependency('accelerate', + version: '>818.60', # vecLib version (TODO: can this be determined?) + modules: [interface: 'lp64'] +) +``` + ## Blocks Enable support for Clang's blocks extension. diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index c45a39f6ddf0..0be1d16ed09c 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -7,6 +7,8 @@ ExternalLibrary, DependencyException, DependencyMethods, BuiltinDependency, SystemDependency, get_leaf_external_dependencies) from .detect import find_external_dependency, get_dep_identifier, packages, _packages_accept_language +from .blas_lapack import openblas_factory + __all__ = [ 'Dependency', @@ -227,6 +229,11 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. 'libssl': 'misc', 'objfw': 'misc', + # From blas_lapack: + 'accelerate': 'blas_lapack', + 'mkl': 'blas_lapack', + 'openblas': 'blas_lapack', + # From platform: 'appleframeworks': 'platform', diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py new file mode 100644 index 000000000000..67817b2b6190 --- /dev/null +++ b/mesonbuild/dependencies/blas_lapack.py @@ -0,0 +1,753 @@ +# Copyright 2013-2020 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import os +from pathlib import Path +import re +import subprocess +import sys +import typing as T + +from .. import mlog +from .. import mesonlib +from ..mesonlib import MachineChoice, OptionKey + +from .base import DependencyMethods, SystemDependency +from .cmake import CMakeDependency +from .detect import packages +from .factory import DependencyFactory, factory_methods +from .pkgconfig import PkgConfigDependency + +if T.TYPE_CHECKING: + from ..environment import Environment + +""" +TODO: how to select BLAS interface layer (LP64, ILP64)? + LP64 is the 32-bit interface (confusingly, despite the "64") + ILP64 is the 64-bit interface + + OpenBLAS library names possible: + - libopenblas + - libopenblas64_ # or nonstandard suffix (see OpenBLAS Makefile) - let's not deal with that + +TODO: do we care here about whether OpenBLAS is compiled with pthreads or OpenMP? + MKL has separate .pc files for this (_seq vs. _iomp), OpenBLAS does not + +TODO: what about blas/lapack vs. cblas/clapack? + See http://nicolas.limare.net/pro/notes/2014/10/31_cblas_clapack_lapacke/ for details + +NOTE: we ignore static libraries for now + +Other Notes: + +- OpenBLAS can be built with NOFORTRAN, in that case it's CBLAS + f2c'd LAPACK +- OpenBLAS can be built without LAPACK support, Arch Linux currently does this + (see https://github.com/scipy/scipy/issues/17465) +- OpenBLAS library can be renamed with an option in its Makefile +- Build options: + - conda-forge: https://github.com/conda-forge/openblas-feedstock/blob/49ca08fc9d1ff220804aa9b894b9a6fe5db45057/recipe/conda_build_config.yaml + - Spack: https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/openblas/package.py#L52 + - vendored into NumPy/SciPy wheels: https://github.com/MacPython/openblas-libs/blob/master/tools/build_openblas.sh#L53 + +Conda-forge library names and pkg-config output +----------------------------------------------- + +(lp64) $ ls lib/libopenblas* +lib/libopenblas.a lib/libopenblasp-r0.3.21.a lib/libopenblasp-r0.3.21.so lib/libopenblas.so lib/libopenblas.so.0 +(lp64) $ ls include/ +cblas.h f77blas.h lapacke_config.h lapacke.h lapacke_mangling.h lapacke_utils.h lapack.h openblas_config.h + +(ilp64) $ ls lib/libopenblas* +lib/libopenblas64_.a lib/libopenblas64_p-r0.3.21.a lib/libopenblas64_p-r0.3.21.so lib/libopenblas64_.so lib/libopenblas64_.so.0 +(ilp64) $ ls include/ +cblas.h f77blas.h lapacke_config.h lapacke.h lapacke_mangling.h lapacke_utils.h lapack.h openblas_config.h + +(lp64) $ pkg-config --cflags openblas +-I/path/to/env/include +(lp64) $ pkg-config --libs openblas +-L/path/to/env/lib -lopenblas +(ilp64) $ pkg-config --cflags openblas +-I/path/to/env/include +ilp64) $ pkg-config --libs openblas +-L/path/to/envlib -lopenblas + + +LP64 vs ILP64 interface and symbol names +---------------------------------------- + +Relevant discussions: +- For OpenBLAS, see https://github.com/xianyi/OpenBLAS/issues/646 for standardized agreement on shared library name and symbol suffix. +- PRs that added support for ILP64 to `numpy.distutils`: + - with `64_` symbol suffix: https://github.com/numpy/numpy/pull/15012 + - generalized to non-suffixed build: https://github.com/numpy/numpy/pull/15069 +- PR that added support for ILP64 to SciPy: https://github.com/scipy/scipy/pull/11302 +- Note to self: when SciPy uses ILP64, it also still requires LP64, because not + all submodules support ILP64. Not for the same extensions though. + +$ objdump -t lp64/lib/libopenblas.so | grep -E "scopy*" # output cleaned up +cblas_scopy +scopy_ + +$ objdump -t ilp64/lib/libopenblas64_.so | grep -E "scopy*" +cblas_scopy64_ +scopy_64_ + +What should be implemented in Meson, and what should be left to users? Thoughts: + +1. Meson should support a keyword to select the desired interface (`interface : + 'ilp64'`), defaulting to `'lp64'` because that is what reference BLAS/LAPACK + provide and what is typically expected. +2. The dependency object returned by `dependency('openblas')` or similar should + be query-able for what the interface is. +3. For OpenBLAS, Meson should look for `libopenblas64_.so` for ILP64. +4. For Fortran, should Meson automatically set the required compile flag + `-fdefault-integer-8`? + - Note: this flag is specific to gfortran and Clang; For Intel compilers it is + `-integer-size 64`(Linux/macOS), `/integer-size: 64` (Windows). + - This is almost always the right thing to do; however for integer + variables that reference an external non-BLAS/LAPACK interface and must + not be changed to 64-bit, those should then be explicitly `integer*4` in + the user code. +5. Users are responsible for implementing name mangling. I.e., appending `64_` + to BLAS/LAPACK symbols when they are requesting ILP64, and also using + portable integer types in their code if they want to be able to build with + both LP64 and ILP64. This typically looks something like: + + ```C + #ifdef HAVE_CBLAS64_ + #define CBLAS_FUNC(name) name ## 64_ + #else + #define CBLAS_FUNC(name) name + #endif + + CBLAS_FUNC(cblas_dgemm)(...); + ``` +6. Users are responsible for implementing a build option (e.g., in `meson_options.txt`) + if they want to allow switching between LP64 and ILP64. +7. Meson doesn't know anything about f2py; users have to instruct f2py to use + 64-bit integers with `--f2cmap` or similar. See `get_f2py_int64_options` in + SciPy for details. +8. When mixing C and Fortran code, the C code typically needs mangling because + Fortran expects a trailing underscore. This is up to the user to implement. +9. TBD: `numpy.distutils` does a symbol prefix/suffix check and provides the + result to its users, it could be helpful if Meson did this. See + https://github.com/numpy/numpy/blob/6094eff9/numpy/distutils/system_info.py#L2271-L2278. + +CBLAS +----- + +Initial rough notes: + +- Not all implementations provide CBLAS, +- The header is typically named `cblas.h`, however MKL calls it `mkl_cblas.h`, +- OpenBLAS can be built without a Fortran compiler, in that case it's CBLAS + f2c'd LAPACK, +- It would be useful if the dependency objects that Meson returned can be + queried for whether CBLAS is present or not. +- numpy.distutils detects CBLAS and defines `HAVE_CBLAS` if it's found. +- BLIS doesn't build the CBLAS interface by default. To build it, define + `BLIS_ENABLE_CBLAS`. +- NumPy requires CBLAS, it's not optional. + +MKL +--- + +MKL docs: https://www.intel.com/content/www/us/en/develop/documentation/onemkl-windows-developer-guide/top/linking-your-application-with-onemkl/linking-in-detail/linking-with-interface-libraries/using-the-ilp64-interface-vs-lp64-interface.html +MKL link line advisor: https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-link-line-advisor.html + +$ cd /opt/intel/oneapi/mkl/latest/lib # from recommended Intel installer +$ ls pkgconfig/ +mkl-dynamic-ilp64-iomp.pc mkl-dynamic-lp64-iomp.pc mkl-static-ilp64-iomp.pc mkl-static-lp64-iomp.pc +mkl-dynamic-ilp64-seq.pc mkl-dynamic-lp64-seq.pc mkl-static-ilp64-seq.pc mkl-static-lp64-seq.pc + +$ pkg-config --libs mkl-dynamic-ilp64-seq +-L/opt/intel/oneapi/mkl/latest/lib/pkgconfig/../../lib/intel64 -lmkl_intel_ilp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl +$ pkg-config --cflags mkl-dynamic-ilp64-seq +-DMKL_ILP64 -I/opt/intel/oneapi/mkl/latest/lib/pkgconfig/../../include + +$ ls intel64/libmkl* +intel64/libmkl_avx2.so.2 intel64/libmkl_cdft_core.so.2 intel64/libmkl_intel_lp64.so.2 intel64/libmkl_sequential.a +intel64/libmkl_avx512.so.2 intel64/libmkl_core.a intel64/libmkl_intel_thread.a intel64/libmkl_sequential.so +intel64/libmkl_avx.so.2 intel64/libmkl_core.so intel64/libmkl_intel_thread.so intel64/libmkl_sequential.so.2 +intel64/libmkl_blacs_intelmpi_ilp64.a intel64/libmkl_core.so.2 intel64/libmkl_intel_thread.so.2 intel64/libmkl_sycl.a +intel64/libmkl_blacs_intelmpi_ilp64.so intel64/libmkl_def.so.2 intel64/libmkl_lapack95_ilp64.a intel64/libmkl_sycl.so +intel64/libmkl_blacs_intelmpi_ilp64.so.2 intel64/libmkl_gf_ilp64.a intel64/libmkl_lapack95_lp64.a intel64/libmkl_sycl.so.2 +intel64/libmkl_blacs_intelmpi_lp64.a intel64/libmkl_gf_ilp64.so intel64/libmkl_mc3.so.2 intel64/libmkl_tbb_thread.a +intel64/libmkl_blacs_intelmpi_lp64.so intel64/libmkl_gf_ilp64.so.2 intel64/libmkl_mc.so.2 intel64/libmkl_tbb_thread.so +intel64/libmkl_blacs_intelmpi_lp64.so.2 intel64/libmkl_gf_lp64.a intel64/libmkl_pgi_thread.a intel64/libmkl_tbb_thread.so.2 +intel64/libmkl_blacs_openmpi_ilp64.a intel64/libmkl_gf_lp64.so intel64/libmkl_pgi_thread.so intel64/libmkl_vml_avx2.so.2 +intel64/libmkl_blacs_openmpi_ilp64.so intel64/libmkl_gf_lp64.so.2 intel64/libmkl_pgi_thread.so.2 intel64/libmkl_vml_avx512.so.2 +intel64/libmkl_blacs_openmpi_ilp64.so.2 intel64/libmkl_gnu_thread.a intel64/libmkl_rt.so intel64/libmkl_vml_avx.so.2 +intel64/libmkl_blacs_openmpi_lp64.a intel64/libmkl_gnu_thread.so intel64/libmkl_rt.so.2 intel64/libmkl_vml_cmpt.so.2 +intel64/libmkl_blacs_openmpi_lp64.so intel64/libmkl_gnu_thread.so.2 intel64/libmkl_scalapack_ilp64.a intel64/libmkl_vml_def.so.2 +# intel64/libmkl_blacs_openmpi_lp64.so.2 intel64/libmkl_intel_ilp64.a intel64/libmkl_scalapack_ilp64.so intel64/libmkl_vml_mc2.so.2 +# intel64/libmkl_blas95_ilp64.a intel64/libmkl_intel_ilp64.so intel64/libmkl_scalapack_ilp64.so.2 intel64/libmkl_vml_mc3.so.2 +# intel64/libmkl_blas95_lp64.a intel64/libmkl_intel_ilp64.so.2 intel64/libmkl_scalapack_lp64.a intel64/libmkl_vml_mc.so.2 +intel64/libmkl_cdft_core.a intel64/libmkl_intel_lp64.a intel64/libmkl_scalapack_lp64.so +intel64/libmkl_cdft_core.so intel64/libmkl_intel_lp64.so intel64/libmkl_scalapack_lp64.so.2 + +$ objdump -t intel64/libmkl_intel_ilp64.so | grep scopy # cleaned up output: +0000000000000000 *UND* 0000000000000000 mkl_blas_scopy +0000000000323000 g F .text 0000000000000030 cblas_scopy_64 +00000000002cecf0 g F .text 0000000000000030 cblas_scopy +000000000025aca0 g F .text 00000000000001d0 mkl_blas__scopy +000000000025aca0 g F .text 00000000000001d0 scopy_64_ +000000000025aca0 g F .text 00000000000001d0 scopy_64 +000000000025aca0 g F .text 00000000000001d0 scopy_ +000000000025aca0 g F .text 00000000000001d0 scopy + +tl;dr: for MKL there's 8 pkg-config file, so you choose lp64/ilp64, dynamic/static, and pthreads/openmp; + after that you can pick whatever symbols you like, they all exist and are aliases. + +A test with SciPy & MKL: + +$ # No pkg-config files for MKL in conda-forge yet, so use the Intel installer: +$ export PKG_CONFIG_PATH=/opt/intel/oneapi/mkl/latest/lib/pkgconfig/ +$ meson setup build --prefix=$PWD/build-install -Dblas=mkl-dynamic-ilp64-seq -Dlapack=mkl-dynamic-ilp64-seq +$ python dev.py build +$ ldd build/scipy/linalg/_flapack.cpython-310-x86_64-linux-gnu.so + linux-vdso.so.1 (0x00007ffe2cd14000) + libmkl_intel_ilp64.so.2 => /opt/intel/oneapi/mkl/latest/lib/intel64/libmkl_intel_ilp64.so.2 (0x00007f75a5000000) + libmkl_sequential.so.2 => /opt/intel/oneapi/mkl/latest/lib/intel64/libmkl_sequential.so.2 (0x00007f75a3400000) + libmkl_core.so.2 => /opt/intel/oneapi/mkl/latest/lib/intel64/libmkl_core.so.2 (0x00007f759f000000) + libm.so.6 => /usr/lib/libm.so.6 (0x00007f75a62e2000) + libc.so.6 => /usr/lib/libc.so.6 (0x00007f759ee19000) + /usr/lib64/ld-linux-x86-64.so.2 (0x00007f75a63e8000) + libdl.so.2 => /usr/lib/libdl.so.2 (0x00007f75a62db000) + libpthread.so.0 => /usr/lib/libpthread.so.0 (0x00007f75a62d6000 +$ # Due to RPATH stripping on install, this doesn't actually work unless we put MKL into our conda env: +$ ldd build-install/lib/python3.10/site-packages/scipy/linalg/_flapack.cpython-310-x86_64-linux-gnu.so + linux-vdso.so.1 (0x00007ffdf6bc6000) + libmkl_intel_ilp64.so.2 => not found + libmkl_sequential.so.2 => not found + libmkl_core.so.2 => not found + libm.so.6 => /usr/lib/libm.so.6 (0x00007f7e35318000) + libc.so.6 => /usr/lib/libc.so.6 (0x00007f7e35131000) + /usr/lib64/ld-linux-x86-64.so.2 (0x00007f7e356e5000) + +Also need to remember that MKL uses a g77 ABI (Accelerate too), while OpenBLAS and most other BLAS +libraries will be using the gfortran ABI. See the `use-g77-abi` option in SciPy's meson_options.txt. + +ArmPL +----- + +Pkg-config file names for ArmPL to use (from https://github.com/spack/spack/pull/34979#discussion_r1073213319): + + armpl-dynamic-ilp64-omp armpl-Fortran-static-ilp64-omp + armpl-dynamic-ilp64-omp.pc armpl-Fortran-static-ilp64-omp.pc + armpl-dynamic-ilp64-seq armpl-Fortran-static-ilp64-seq + armpl-dynamic-ilp64-seq.pc armpl-Fortran-static-ilp64-seq.pc + armpl-dynamic-lp64-omp armpl-Fortran-static-lp64-omp + armpl-dynamic-lp64-omp.pc armpl-Fortran-static-lp64-omp.pc + armpl-dynamic-lp64-seq armpl-Fortran-static-lp64-seq + armpl-dynamic-lp64-seq.pc armpl-Fortran-static-lp64-seq.pc + armpl-Fortran-dynamic-ilp64-omp armpl-static-ilp64-omp + armpl-Fortran-dynamic-ilp64-omp.pc armpl-static-ilp64-omp.pc + armpl-Fortran-dynamic-ilp64-seq armpl-static-ilp64-seq + armpl-Fortran-dynamic-ilp64-seq.pc armpl-static-ilp64-seq.pc + armpl-Fortran-dynamic-lp64-omp armpl-static-lp64-omp + armpl-Fortran-dynamic-lp64-omp.pc armpl-static-lp64-omp.pc + armpl-Fortran-dynamic-lp64-seq armpl-static-lp64-seq + armpl-Fortran-dynamic-lp64-seq.pc armpl-static-lp64-seq.pc + +Note that as of Jan'23, ArmPL ships the files without a `.pc` extension (that +will hopefully be fixed), and Spack renames then so adds the `.pc` copies of +the original files. + +Apple Accelerate +---------------- + +In macOS >=13.3, two LP64 and one ILP64 build of vecLib are shipped. Due to +compatibility, the legacy interfaces (providing LAPACK 3.2.1) will be used by +default. To use the new interfaces (providing LAPACK 3.9.1), including ILP64, +it is necessary to set some #defines before including Accelerate / vecLib headers: + +- `-DACCELERATE_NEW_LAPACK`: use the new interfaces +- `-DACCELERATE_LAPACK_ILP64`: use new ILP64 interfaces (note this requires + `-DACCELERATE_NEW_LAPACK` to be set as well) + +The normal F77 symbols will remain as the legacy implementation. The newer +interfaces have separate symbols with suffixes `$NEWLAPACK` or `$NEWLAPACK$ILP64`. + +Example binary symbols: + +- `_dgemm_`: this is the legacy implementation +- `_dgemm$NEWLAPACK`: this is the new implementation +- `_dgemm$NEWLAPACK$ILP64`: this is the new ILP64 implementaion + +If you use Accelerate / vecLib headers with the above defines, you don't need +to worry about the symbol names. They'll get aliased correctly. + +For headers and linker flags, check if these directories exist before using them: + +1. `-I/System/Library/Frameworks/vecLib.framework/Headers`, + flags: ['-Wl,-framework', '-Wl,Accelerate'] +2. `-I/System/Library/Frameworks/vecLib.framework/Headers`, + flags: ['-Wl,-framework', '-Wl,vecLib'] + +Note that the dylib's are no longer physically present, they're provided in the +shared linker cache. +""" + + +class BLASLAPACKMixin(): + def parse_modules(self, kwargs: T.Dict[str, T.Any]) -> None: + modules: T.List[str] = mesonlib.extract_as_list(kwargs, 'modules') + valid_modules = ['interface: lp64', 'interface: ilp64', 'cblas', 'lapack', 'lapacke'] + for module in modules: + if module not in valid_modules: + raise mesonlib.MesonException(f'Unknown modules argument: {module}') + + interface = [s for s in modules if s.startswith('interface')] + if interface: + if len(interface) > 1: + raise mesonlib.MesonException(f'Only one interface must be specified, found: {interface}') + self.interface = interface[0].split(' ')[1] + else: + self.interface = 'lp64' + + self.needs_cblas = 'cblas' in modules + self.needs_lapack = 'lapack' in modules + self.needs_lapacke = 'lapacke' in modules + + def check_symbols(self, compile_args, suffix=None) -> None: + # verify that we've found the right LP64/ILP64 interface + symbols = ['dgemm_'] + if self.needs_cblas: + symbols += ['cblas_dgemm'] + if self.needs_lapack: + symbols += ['zungqr_'] + if self.needs_lapacke: + symbols += ['LAPACKE_zungqr'] + + if suffix is None: + suffix = self.get_symbol_suffix() + + prototypes = "".join(f"void {symbol}{suffix}();\n" for symbol in symbols) + calls = " ".join(f"{symbol}{suffix}();\n" for symbol in symbols) + code = (f"{prototypes}" + "int main(int argc, const char *argv[])\n" + "{\n" + f" {calls}" + " return 0;\n" + "}" + ) + code = '''#ifdef __cplusplus + extern "C" { + #endif + ''' + code + ''' + #ifdef __cplusplus + } + #endif + ''' + + return self.clib_compiler.links(code, self.env, extra_args=compile_args)[0] + + def get_variable(self, **kwargs: T.Dict[str, T.Any]) -> str: + # TODO: what's going on with `get_variable`? Need to pick from + # cmake/pkgconfig/internal/..., but not system? + varname = kwargs['pkgconfig'] + if varname == 'interface': + return self.interface + elif varname == 'symbol_suffix': + return self.get_symbol_suffix() + return super().get_variable(**kwargs) + + +class OpenBLASMixin(): + def get_symbol_suffix(self) -> str: + return '' if self.interface == 'lp64' else self._ilp64_suffix + + def probe_symbols(self, compile_args) -> bool: + """There are two common ways of building ILP64 BLAS, check which one we're dealing with""" + if self.interface == 'lp64': + return self.check_symbols(compile_args) + + if self.check_symbols(compile_args, '64_'): + self._ilp64_suffix = '64_' + elif self.check_symbols(compile_args, ''): + self._ilp64_suffix = '' + else: + return False + return True + + +class OpenBLASSystemDependency(BLASLAPACKMixin, OpenBLASMixin, SystemDependency): + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + # First, look for paths specified in a machine file + props = self.env.properties[self.for_machine].properties + if any(x in props for x in ['openblas_includedir', 'openblas_librarydir']): + self.detect_openblas_machine_file(props) + + # Then look in standard directories by attempting to link + if not self.is_found: + extra_libdirs: T.List[str] = [] + self.detect(extra_libdirs) + + if self.is_found: + self.version = self.detect_openblas_version() + + def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[T.List[str]] = None) -> None: + if lib_dirs is None: + lib_dirs = [] + if inc_dirs is None: + inc_dirs = [] + + if self.interface == 'lp64': + libnames = ['openblas'] + elif self.interface == 'ilp64': + libnames = ['openblas64', 'openblas_ilp64', 'openblas'] + + for libname in libnames: + link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in ['openblas_config.h', 'openblas/openblas_config.h']: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + self._openblas_config_header = hdr + break + + if link_arg and found_header: + if not self.probe_symbols(link_arg): + continue + self.is_found = True + self.link_args += link_arg + self.compile_args += incdir_args + break + + def detect_openblas_machine_file(self, props: dict) -> None: + # TBD: do we need to support multiple extra dirs? + incdir = props.get('openblas_includedir') + assert incdir is None or isinstance(incdir, str) + libdir = props.get('openblas_librarydir') + assert libdir is None or isinstance(libdir, str) + + if incdir and libdir: + self.is_found = True + if not Path(incdir).is_absolute() or not Path(libdir).is_absolute(): + raise mesonlib.MesonException('Paths given for openblas_includedir and ' + 'openblas_librarydir in machine file must be absolute') + elif incdir or libdir: + raise mesonlib.MesonException('Both openblas_includedir *and* openblas_librarydir ' + 'have to be set in your machine file (one is not enough)') + else: + raise mesonlib.MesonBugException('issue with openblas dependency detection, should not ' + 'be possible to reach this else clause') + + self.detect([libdir], [incdir]) + + def detect_openblas_version(self) -> str: + v, _ = self.clib_compiler.get_define('OPENBLAS_VERSION', + f'#include <{self._openblas_config_header}>', + self.env, [], [self]) + + m = re.search(r'\d+(?:\.\d+)+', v) + if not m: + mlog.debug('Failed to extract openblas version information') + return None + return m.group(0) + + +class OpenBLASPkgConfigDependency(BLASLAPACKMixin, OpenBLASMixin, PkgConfigDependency): + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + if self.interface == 'lp64' and name != 'openblas': + # Check only for 'openblas' for LP64 (there are multiple names for ILP64) + self.is_found = False + return None + + super().__init__(name, env, kwargs) + + if not self.probe_symbols(self.link_args): + self.is_found = False + + +class OpenBLASCMakeDependency(BLASLAPACKMixin, OpenBLASMixin, CMakeDependency): + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], + language: T.Optional[str] = None, force_use_global_compilers: bool = False) -> None: + super().__init__('OpenBLAS', env, kwargs, language, force_use_global_compilers) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + if self.interface == 'ilp64': + self.is_found = False + elif not self.probe_symbols(self.link_args): + self.is_found = False + + +class NetlibPkgConfigDependency(BLASLAPACKMixin, PkgConfigDependency): + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + # TODO: add 'cblas' + super().__init__('blas', env, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + def get_symbol_suffix(self) -> str: + return '' + + +class AccelerateSystemDependency(BLASLAPACKMixin, SystemDependency): + """ + Accelerate is always installed on macOS, and not available on other OSes. + We only support using Accelerate on macOS >=13.3, where Apple shipped a + major update to Accelerate, fixed a lot of bugs, and bumped the LAPACK + version from 3.2 to 3.9. The older Accelerate version is still available, + and can be obtained as a standard Framework dependency with one of: + + dependency('Accelerate') + dependency('appleframeworks', modules : 'Accelerate') + + """ + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + for_machine = MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST + if environment.machines[for_machine].is_darwin() and self.check_macOS_recent_enough(): + self.detect(kwargs) + + def check_macOS_recent_enough(self) -> bool: + cmd = ['xcrun', '-sdk', 'macosx', '--show-sdk-version'] + sdk_version = str(subprocess.run(cmd, capture_output=True, check=True).stdout) + return sdk_version >= '13.3' + + def detect(self, kwargs: T.Dict[str, T.Any]) -> None: + from .framework import ExtraFrameworkDependency + dep = ExtraFrameworkDependency('Accelerate', self.env, kwargs) + self.is_found = dep.is_found + if self.is_found: + self.compile_args = dep.compile_args + self.link_args = dep.link_args + self.compile_args += ['-DACCELERATE_NEW_LAPACK'] + if self.interface == 'ilp64': + self.compile_args += ['-DACCELERATE_LAPACK_ILP64'] + + # We won't check symbols here, because Accelerate is built in a consistent fashion + # with known symbol mangling, unlike OpenBLAS or Netlib BLAS/LAPACK. + return None + + def get_symbol_suffix(self) -> str: + return '$NEWLAPACK' if self.interface == 'lp64' else '$NEWLAPACK$ILP64' + + +class MKLMixin(): + def get_symbol_suffix(self) -> str: + return '' if self.interface == 'lp64' else '_64' + + def parse_mkl_options(self, kwargs: T.Dict[str, T.Any]) -> None: + """Parse `modules` and remove threading and SDL options from it if they are present. + + Removing 'threading: ' and 'sdl' from `modules` is needed to ensure those + don't get to the generic parse_modules() method for all BLAS/LAPACK dependencies. + """ + modules: T.List[str] = mesonlib.extract_as_list(kwargs, 'modules') + threading_module = [s for s in modules if s.startswith('threading')] + sdl_module = [s for s in modules if s.startswith('sdl')] + + if not threading_module: + self.threading = 'iomp' + elif len(threading_module) > 1: + raise mesonlib.MesonException(f'Multiple threading arguments: {threading_modules}') + else: + # We have a single threading option specified - validate and process it + opt = threading_module[0] + if opt not in ['threading: ' + s for s in ('seq', 'iomp', 'gomp', 'tbb')]: + raise mesonlib.MesonException(f'Invalid threading argument: {opt}') + + self.threading = opt.split(' ')[1] + modules = [s for s in modules if not s == opt] + kwargs['modules'] = modules + + if not sdl_module: + self.use_sdl = 'auto' + elif len(sdl_module) > 1: + raise mesonlib.MesonException(f'Multiple sdl arguments: {threading_modules}') + else: + # We have a single sdl option specified - validate and process it + opt = sdl_module[0] + if opt not in ['sdl: ' + s for s in ('true', 'false', 'auto')]: + raise mesonlib.MesonException(f'Invalid sdl argument: {opt}') + + self.use_sdl = { + 'false': False, + 'true': True, + 'auto': 'auto' + }.get(opt.split(' ')[1]) + modules = [s for s in modules if not s == opt] + kwargs['modules'] = modules + + # Parse common BLAS/LAPACK options + self.parse_modules(kwargs) + + # Check if we don't have conflicting options between SDL's defaults and interface/threading + self.sdl_default_opts = self.interface == 'lp64' and self.threading == 'iomp' + if self.use_sdl == 'auto' and not self.sdl_default_opts: + self.use_sdl = False + elif self.use_sdl and not self.sdl_default_opts: + # If we're here, we got an explicit `sdl: 'true'` + raise mesonlib.MesonException(f'Linking SDL implies using LP64 and Intel OpenMP, found ' + f'conflicting options: {self.interface}, {self.threading}') + + return None + + +class MKLPkgConfigDependency(BLASLAPACKMixin, MKLMixin, PkgConfigDependency): + """ + pkg-config files for MKL were fixed recently, and should work from 2023.0 + onwards. Directly using a specific one like so should work: + + dependency('mkl-dynamic-ilp64-seq') + + The naming scheme is `mkl-libtype-interface-threading.pc`, with values: + - libtype: dynamic/static + - interface: lp64/ilp64 + - threading: seq/iomp/gomp/tbb + + Furthermore there is a pkg-config file for the Single Dynamic Library + (libmkl_rt.so) named `mkl-sdl.pc` (only added in 2023.0). + + Note that there is also an MKLPkgConfig dependency in scalapack.py, which + has more manual fixes. + """ + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + self.feature_since = ('1.3.0', '') + self.parse_mkl_options(kwargs) + if self.use_sdl == 'auto': + # Layered libraries are preferred, and .pc files for layered were + # available before the .pc file for SDL + self.use_sdl = False + + static_opt = kwargs.get('static', env.coredata.get_option(OptionKey('prefer_static'))) + libtype = 'static' if static_opt else 'dynamic' + + if self.use_sdl: + name = 'mkl-sdl' + else: + name = f'mkl-{libtype}-{self.interface}-{self.threading}' + super().__init__(name, env, kwargs) + + +class MKLSystemDependency(BLASLAPACKMixin, MKLMixin, SystemDependency): + """This only detects MKL's Single Dynamic Library (SDL)""" + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_mkl_options(kwargs) + if self.use_sdl == 'auto': + # Too complex to use layered here - only supported with pkg-config + self.use_sdl = True + + if self.use_sdl: + self.detect_sdl() + return None + + def detect_sdl(self) -> None: + # Use MKLROOT in addition to standard libdir(s) + _m = os.environ.get('MKLROOT') + mklroot = Path(_m).resolve() if _m else None + lib_dirs = [] + inc_dirs = [] + if mklroot is not None: + libdir = mklroot / 'lib' / 'intel64' + if not libdir.exists(): + # MKLROOT may be pointing at the prefix where MKL was installed from PyPI + # or Conda (Intel supports those install methods, but dropped the `intel64` + # part, libraries go straight into /lib + libdir = mklroot / 'lib' + incdir = mklroot / 'include' + lib_dirs += [libdir] + inc_dirs += [incdir] + if not libdir.exists() or not incdir.exists(): + mlog.warning(f'MKLROOT env var set to {mklroot}, but not pointing to an MKL install') + + link_arg = self.clib_compiler.find_library('mkl_rt', self.env, lib_dirs) + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + found_header, _ = self.clib_compiler.has_header('mkl_version.h', '', self.env, + dependencies=[self], extra_args=incdir_args) + if link_arg and found_header: + self.is_found = True + self.compile_args += incdir_args + self.link_args += link_arg + if not sys.platform == 'win32': + self.link_args += ['-lpthread', '-lm', '-ldl'] + + # Determine MKL version + ver, _ = self.clib_compiler.get_define('INTEL_MKL_VERSION', + '#include "mkl_version.h"', + self.env, + dependencies=[self], + extra_args=incdir_args) + if len(ver) == 8: + year = ver[:4] + minor = str(int(ver[4:6])) + update = str(int(ver[6:])) + # Note: this is the order as of 2023.2.0, but it looks the wrong way around + # (INTEL_MKL_VERSION is defined as 20230002 in that release), could be swapped in the future perhaps + self.version = f'{year}.{update}.{minor}' + else: + mlog.warning(f'MKL version detection issue, found {ver}') + + +@factory_methods({DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM, DependencyMethods.CMAKE}) +def openblas_factory(env: 'Environment', for_machine: 'MachineChoice', + kwargs: T.Dict[str, T.Any], + methods: T.List[DependencyMethods]) -> T.List['DependencyGenerator']: + candidates: T.List['DependencyGenerator'] = [] + + if DependencyMethods.PKGCONFIG in methods: + for pkg in ['openblas64', 'openblas_ilp64', 'openblas']: + candidates.append(functools.partial( + OpenBLASPkgConfigDependency, pkg, env, kwargs)) + + if DependencyMethods.SYSTEM in methods: + candidates.append(functools.partial( + OpenBLASSystemDependency, 'openblas', env, kwargs)) + + if DependencyMethods.CMAKE in methods: + candidates.append(functools.partial( + OpenBLASCMakeDependency, 'OpenBLAS', env, kwargs)) + + return candidates + +packages['openblas'] = openblas_factory + + +packages['netlib-blas'] = netlib_factory = DependencyFactory( + 'netlib-blas', + [DependencyMethods.PKGCONFIG], #, DependencyMethods.SYSTEM], + #system_class=NetlibSystemDependency, + pkgconfig_class=NetlibPkgConfigDependency, +) + + +packages['accelerate'] = accelerate_factory = DependencyFactory( + 'accelerate', + [DependencyMethods.SYSTEM], + system_class=AccelerateSystemDependency, +) + + +packages['mkl'] = mkl_factory = DependencyFactory( + 'mkl', + [DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM], + pkgconfig_class=MKLPkgConfigDependency, + system_class=MKLSystemDependency, +) diff --git a/test cases/frameworks/38 blas-lapack/cblas_lapack.c b/test cases/frameworks/38 blas-lapack/cblas_lapack.c new file mode 100644 index 000000000000..3ebdccb7a43f --- /dev/null +++ b/test cases/frameworks/38 blas-lapack/cblas_lapack.c @@ -0,0 +1,47 @@ +// Basic BLAS/LAPACK example adapted from a test in Spack for OpenBLAS + +#include "cblas_lapack.h" + +int main(void) { + // CBLAS: + blas_int n_elem = 9; + blas_int incx = 1; + double A[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; + double B[6] = {1.0, 2.0, 1.0, -3.0, 4.0, -1.0}; + double C[9] = {.5, .5, .5, .5, .5, .5, .5, .5, .5}; + double norm; + + CBLAS_FUNC(cblas_dgemm) + (CblasColMajor, CblasNoTrans, CblasTrans, 3, 3, 2, 1, A, 3, B, 3, 2, C, 3); + norm = CBLAS_FUNC(cblas_dnrm2)(n_elem, C, incx) - 28.017851; + + if (fabs(norm) < 1e-5) { + printf("OK: CBLAS result using dgemm and dnrm2 as expected\n"); + } else { + fprintf(stderr, "CBLAS result using dgemm and dnrm2 incorrect: %f\n", norm); + exit(EXIT_FAILURE); + } + + // LAPACK: + double m[] = {3, 1, 3, 1, 5, 9, 2, 6, 5}; + double x[] = {-1, 3, -3}; + lapack_int ipiv[3]; + lapack_int info; + lapack_int n = 1; + lapack_int nrhs = 1; + lapack_int lda = 3; + lapack_int ldb = 3; + + LAPACK_FUNC(dgesv)(&n, &nrhs, &m[0], &lda, ipiv, &x[0], &ldb, &info); + n_elem = 3; + norm = CBLAS_FUNC(cblas_dnrm2)(n_elem, x, incx) - 4.255715; + + if (fabs(norm) < 1e-5) { + printf("OK: LAPACK result using dgesv as expected\n"); + } else { + fprintf(stderr, "LAPACK result using dgesv incorrect: %f\n", norm); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/test cases/frameworks/38 blas-lapack/cblas_lapack.h b/test cases/frameworks/38 blas-lapack/cblas_lapack.h new file mode 100644 index 000000000000..5b8ddc023d9d --- /dev/null +++ b/test cases/frameworks/38 blas-lapack/cblas_lapack.h @@ -0,0 +1,113 @@ +// Unified CBLAS and LAPACK header to account for the various name mangling +// schemes used by Accelerate, OpenBLAS, MKL & co. As well as for the names of +// header files varying (e.g. cblas.h, Accelerate/Accelerate.h, mkl_cblas.h) +// +// Accelerate.h (via cblas_new.h) contains non-suffixed names, and the suffixed +// symbols are added via aliases inserted with __asm. +// OpenBLAS headers do contain suffixed names (e.g., `cblas_dgemm64_`) +// The end result after the standardization discussion in +// https://github.com/Reference-LAPACK/lapack/issues/666 should be +// `cblas_dgemm_64`, however that isn't yet final or implemented. +// +// # Actual symbols present in MKL: + +// 00000000003e4970 T cblas_dgemm +// 00000000003e4970 T cblas_dgemm_ +// 00000000003e34e0 T cblas_dgemm_64 +// 00000000003e34e0 T cblas_dgemm_64_ +// +// 00000000004e9f80 T dgesv +// 00000000004e9f80 T dgesv_ +// 00000000004ea050 T dgesv_64 +// 00000000004ea050 T dgesv_64_ +// +// +// # Actual symbols present in OpenBLAS (in `libopenblas64`): +// +// 00000000000a3430 T cblas_dgemm64_ +// +// 00000000000a6e50 T dgesv_64_ + +// Name mangling adapted/extended from NumPy's npy_cblas.h +#include +#include +#include + +#ifdef ACCELERATE_NEW_LAPACK +#if __MAC_OS_X_VERSION_MAX_ALLOWED < 130300 +#ifdef HAVE_BLAS_ILP64 +#error "Accelerate ILP64 support is only available with macOS 13.3 SDK or later" +#endif +#else +#define NO_APPEND_FORTRAN +#ifdef HAVE_BLAS_ILP64 +#define BLAS_SYMBOL_SUFFIX $NEWLAPACK$ILP64 +#else +#define BLAS_SYMBOL_SUFFIX $NEWLAPACK +#endif +#endif +#endif + +#ifdef NO_APPEND_FORTRAN +#define BLAS_FORTRAN_SUFFIX +#else +#define BLAS_FORTRAN_SUFFIX _ +#endif + +#ifndef BLAS_SYMBOL_SUFFIX +#define BLAS_SYMBOL_SUFFIX +#endif + +#define BLAS_FUNC_CONCAT(name, suffix, suffix2) name##suffix##suffix2 +#define BLAS_FUNC_EXPAND(name, suffix, suffix2) \ + BLAS_FUNC_CONCAT(name, suffix, suffix2) + +#define CBLAS_FUNC(name) BLAS_FUNC_EXPAND(name, , BLAS_SYMBOL_SUFFIX) +#ifdef OPENBLAS_ILP64_NAMING_SCHEME +#define BLAS_FUNC(name) \ + BLAS_FUNC_EXPAND(name, BLAS_FORTRAN_SUFFIX, BLAS_SYMBOL_SUFFIX) +#define LAPACK_FUNC(name) BLAS_FUNC(name) +#else +#define BLAS_FUNC(name) \ + BLAS_FUNC_EXPAND(name, BLAS_SYMBOL_SUFFIX, BLAS_FORTRAN_SUFFIX) +#define LAPACK_FUNC(name) BLAS_FUNC(name) +#endif + +#ifdef HAVE_BLAS_ILP64 +#define blas_int long +#define lapack_int long +#else +#define blas_int int +#define lapack_int int +#endif + +enum CBLAS_ORDER { CblasRowMajor = 101, CblasColMajor = 102 }; +enum CBLAS_TRANSPOSE { + CblasNoTrans = 111, + CblasTrans = 112, + CblasConjTrans = 113 +}; + +#ifdef __cplusplus +extern "C" { +#endif + +void CBLAS_FUNC(cblas_dgemm)(const enum CBLAS_ORDER Order, + const enum CBLAS_TRANSPOSE TransA, + const enum CBLAS_TRANSPOSE TransB, + const blas_int M, const blas_int N, + const blas_int K, const double alpha, + const double *A, const blas_int lda, + const double *B, const blas_int ldb, + const double beta, double *C, const blas_int ldc); + +double CBLAS_FUNC(cblas_dnrm2)(const blas_int N, const double *X, + const blas_int incX); + +void LAPACK_FUNC(dgesv)(lapack_int *n, lapack_int *nrhs, double *a, + lapack_int *lda, lapack_int *ipivot, double *b, + lapack_int *ldb, lapack_int *info); + +#ifdef __cplusplus +} +#endif diff --git a/test cases/frameworks/38 blas-lapack/main.f90 b/test cases/frameworks/38 blas-lapack/main.f90 new file mode 100644 index 000000000000..9abfc0676ef2 --- /dev/null +++ b/test cases/frameworks/38 blas-lapack/main.f90 @@ -0,0 +1,25 @@ +! minimal BLAS test +program AHH + +implicit none +integer :: n, incx, xs +real :: multval +integer :: x(4) +external sccal + +! A very simple test case, scalar x vector +n = 4 +multval = 3.0 +incx = 1 +x = [1, 2, 3, 4] + +call sscal(n, multval, x, incx) + +xs = int(sum(x)) + +if (xs == 30) then + print '("OK: BLAS sum of scaled 1-D array = ",i2)', xs +else + print '("NOK: BLAS sum of scaled 1-D array = ",i2)', xs +end if +end diff --git a/test cases/frameworks/38 blas-lapack/meson.build b/test cases/frameworks/38 blas-lapack/meson.build new file mode 100644 index 000000000000..9decb38c2507 --- /dev/null +++ b/test cases/frameworks/38 blas-lapack/meson.build @@ -0,0 +1,100 @@ +project('test blas and lapack', 'c') + +# OpenBLAS LP64 and ILP64 +openblas_dep = dependency('openblas', + modules: [ + 'interface: lp64', + 'cblas', + 'lapack', + ], + required: false, +) +openblas64_dep = dependency('openblas', + modules: [ + 'interface: ilp64', + 'cblas', + 'lapack', + ], + required: false, +) + +# Accelerate LP64 and ILP64 +accelerate_dep = dependency('accelerate', + modules: [ + 'interface: lp64', + 'cblas', + 'lapack', + ], + required: false, +) +accelerate64_dep = dependency('accelerate', + modules: [ + 'interface: ilp64', + 'cblas', + 'lapack', + ], + required: false, +) + +# MKL LP64 and ILP64 +mkl_dep = dependency('mkl', + modules: [ + 'interface: lp64', + 'cblas', + 'lapack', + ], + required: false, +) +mkl64_dep = dependency('mkl', + modules: [ + 'interface: ilp64', + 'cblas', + 'lapack', + ], + required: false, +) + + +if not openblas_dep.found() and not openblas64_dep.found() and not accelerate_dep.found() and not mkl_dep.found() + error('MESON_SKIP_TEST: no BLAS/LAPACK libraries available') +endif + +deps = { + 'openblas': openblas_dep, + 'openblas64': openblas64_dep, + 'accelerate': accelerate_dep, + 'accelerate64': accelerate64_dep, + 'mkl': mkl_dep, + 'mkl64': mkl64_dep, +} + +foreach name, dep : deps + if dep.found() + blas_interface = dep.get_variable('interface') + symbol_suffix = dep.get_variable('symbol_suffix') + blas_c_args = ['-DBLAS_SYMBOL_SUFFIX=' + symbol_suffix] + if blas_interface == 'ilp64' + blas_c_args += ['-DHAVE_BLAS_ILP64'] + if 'openblas' in dep.name() + blas_c_args += ['-DOPENBLAS_ILP64_NAMING_SCHEME'] + endif + elif blas_interface != 'lp64' + error(f'no interface var for %name% dependency!') + endif + + c_exe = executable('cblas_lapack_c_' + blas_interface, + 'cblas_lapack.c', + c_args: blas_c_args, + dependencies: [dep] + ) + test('cblas_lapack_dep_c', c_exe, timeout: 30) + + if add_languages('fortran', required: false) + f_exe = executable('cblas_lapack_fortran_' + blas_interface, + 'main.f90', + dependencies: [dep] + ) + test('cblas_lapack_dep_fortran', f_exe, timeout: 30) + endif + endif +endforeach diff --git a/test cases/frameworks/38 blas-lapack/test.json b/test cases/frameworks/38 blas-lapack/test.json new file mode 100644 index 000000000000..0c40573168a4 --- /dev/null +++ b/test cases/frameworks/38 blas-lapack/test.json @@ -0,0 +1,3 @@ +{ + "skip_on_jobname": ["azure", "bionic", "cygwin", "fedora", "msys2", "opensuse"] +} From 3b4f3748a0bf79b4250a5b9f7eb39bbd2db467ed Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 10 Oct 2023 16:53:58 +0200 Subject: [PATCH 20/25] BUG: fix macOS SDK version detection --- mesonbuild/dependencies/blas_lapack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index 67817b2b6190..eb89044770bd 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -526,7 +526,7 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. def check_macOS_recent_enough(self) -> bool: cmd = ['xcrun', '-sdk', 'macosx', '--show-sdk-version'] - sdk_version = str(subprocess.run(cmd, capture_output=True, check=True).stdout) + sdk_version = subprocess.run(cmd, capture_output=True, check=True, text=True).stdout.strip() return sdk_version >= '13.3' def detect(self, kwargs: T.Dict[str, T.Any]) -> None: From b008f541b0914a2f07a92cd939d9e1e42f2d78d5 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Mon, 30 Oct 2023 22:23:36 +0100 Subject: [PATCH 21/25] Improve macOS version check in order to use Accelerate NEWLAPACK --- mesonbuild/dependencies/blas_lapack.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index eb89044770bd..42fe4bea93d5 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -15,6 +15,7 @@ import functools import os from pathlib import Path +import platform import re import subprocess import sys @@ -525,9 +526,12 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. self.detect(kwargs) def check_macOS_recent_enough(self) -> bool: + # We need the SDK to be >=13.3 (meaning at least XCode 14.3) cmd = ['xcrun', '-sdk', 'macosx', '--show-sdk-version'] sdk_version = subprocess.run(cmd, capture_output=True, check=True, text=True).stdout.strip() - return sdk_version >= '13.3' + macos_version = platform.mac_ver()[0] + deploy_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', macos_version) + return sdk_version >= '13.3' and deploy_target >= '13.3' def detect(self, kwargs: T.Dict[str, T.Any]) -> None: from .framework import ExtraFrameworkDependency From 3fbdf0b562817422b37cb41fd35196dac3fecea8 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 1 Nov 2023 18:34:46 +0100 Subject: [PATCH 22/25] Add Netlib BLAS (libblas/libcblas) and LAPACK (liblapack/liblapacke) dependencies Also a bit of refactoring and improvements to OpenBLAS and MKL logic to reduce the amount of unnecessary checks. --- mesonbuild/dependencies/__init__.py | 2 + mesonbuild/dependencies/blas_lapack.py | 314 +++++++++++++++++++++---- 2 files changed, 271 insertions(+), 45 deletions(-) diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index 0be1d16ed09c..196d990c63ec 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -231,6 +231,8 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. # From blas_lapack: 'accelerate': 'blas_lapack', + 'blas': 'blas_lapack', + 'lapack': 'blas_lapack', 'mkl': 'blas_lapack', 'openblas': 'blas_lapack', diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index 42fe4bea93d5..92c24725a054 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -302,6 +302,28 @@ """ +def check_blas_machine_file(self, name: str, props: dict) -> T.Tuple[bool, T.List[str]]: + # TBD: do we need to support multiple extra dirs? + incdir = props.get(f'{name}_includedir') + assert incdir is None or isinstance(incdir, str) + libdir = props.get(f'{name}_librarydir') + assert libdir is None or isinstance(libdir, str) + + if incdir and libdir: + has_dirs = True + if not Path(incdir).is_absolute() or not Path(libdir).is_absolute(): + raise mesonlib.MesonException('Paths given for openblas_includedir and ' + 'openblas_librarydir in machine file must be absolute') + return has_dirs, [libdir, incdir] + elif incdir or libdir: + raise mesonlib.MesonException('Both openblas_includedir *and* openblas_librarydir ' + 'have to be set in your machine file (one is not enough)') + else: + raise mesonlib.MesonBugException('issue with openblas dependency detection, should not ' + 'be possible to reach this else clause') + return (False, []) + + class BLASLAPACKMixin(): def parse_modules(self, kwargs: T.Dict[str, T.Any]) -> None: modules: T.List[str] = mesonlib.extract_as_list(kwargs, 'modules') @@ -322,14 +344,17 @@ def parse_modules(self, kwargs: T.Dict[str, T.Any]) -> None: self.needs_lapack = 'lapack' in modules self.needs_lapacke = 'lapacke' in modules - def check_symbols(self, compile_args, suffix=None) -> None: + def check_symbols(self, compile_args, suffix=None, check_cblas=True, + check_lapacke=True, lapack_only=False) -> None: # verify that we've found the right LP64/ILP64 interface - symbols = ['dgemm_'] - if self.needs_cblas: + symbols = [] + if not lapack_only: + symbols += ['dgemm_'] + if check_cblas and self.needs_cblas: symbols += ['cblas_dgemm'] if self.needs_lapack: symbols += ['zungqr_'] - if self.needs_lapacke: + if check_lapacke and self.needs_lapacke: symbols += ['LAPACKE_zungqr'] if suffix is None: @@ -416,13 +441,14 @@ def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[ for libname in libnames: link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) - incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] - for hdr in ['openblas_config.h', 'openblas/openblas_config.h']: - found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], - extra_args=incdir_args) - if found_header: - self._openblas_config_header = hdr - break + if link_arg: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in ['openblas_config.h', 'openblas/openblas_config.h']: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + self._openblas_config_header = hdr + break if link_arg and found_header: if not self.probe_symbols(link_arg): @@ -433,25 +459,10 @@ def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[ break def detect_openblas_machine_file(self, props: dict) -> None: - # TBD: do we need to support multiple extra dirs? - incdir = props.get('openblas_includedir') - assert incdir is None or isinstance(incdir, str) - libdir = props.get('openblas_librarydir') - assert libdir is None or isinstance(libdir, str) - - if incdir and libdir: - self.is_found = True - if not Path(incdir).is_absolute() or not Path(libdir).is_absolute(): - raise mesonlib.MesonException('Paths given for openblas_includedir and ' - 'openblas_librarydir in machine file must be absolute') - elif incdir or libdir: - raise mesonlib.MesonException('Both openblas_includedir *and* openblas_librarydir ' - 'have to be set in your machine file (one is not enough)') - else: - raise mesonlib.MesonBugException('issue with openblas dependency detection, should not ' - 'be possible to reach this else clause') - - self.detect([libdir], [incdir]) + has_dirs, _dirs = check_blas_machine_file('openblas', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) def detect_openblas_version(self) -> str: v, _ = self.clib_compiler.get_define('OPENBLAS_VERSION', @@ -476,7 +487,7 @@ def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> super().__init__(name, env, kwargs) - if not self.probe_symbols(self.link_args): + if self.is_found and not self.probe_symbols(self.link_args): self.is_found = False @@ -489,19 +500,224 @@ def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any], if self.interface == 'ilp64': self.is_found = False - elif not self.probe_symbols(self.link_args): + elif self.is_found and not self.probe_symbols(self.link_args): self.is_found = False -class NetlibPkgConfigDependency(BLASLAPACKMixin, PkgConfigDependency): +class NetlibMixin(): + def get_symbol_suffix(self) -> str: + self._ilp64_suffix = '' # Handle `64_` suffix, or custom suffixes? + return '' if self.interface == 'lp64' else self._ilp64_suffix + + def probe_symbols(self, compile_args, check_cblas=True, check_lapacke=True, + lapack_only=False) -> bool: + """Most ILP64 BLAS builds will not use a suffix, but the new standard will be _64 + (see Reference-LAPACK/lapack#666). Check which one we're dealing with""" + if self.interface == 'lp64': + return self.check_symbols(compile_args, check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only) + + if self.check_symbols(compile_args, '_64', check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only): + self._ilp64_suffix = '_64' + elif self.check_symbols(compile_args, '', check_cblas=check_cblas, + check_lapacke=check_lapacke, lapack_only=lapack_only): + self._ilp64_suffix = '' + else: + return False + return True + + +class NetlibBLASPkgConfigDependency(BLASLAPACKMixin, NetlibMixin, PkgConfigDependency): def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: - # TODO: add 'cblas' - super().__init__('blas', env, kwargs) + # TODO: add ILP64 - needs factory function like for OpenBLAS + super().__init__(name, env, kwargs) self.feature_since = ('1.3.0', '') self.parse_modules(kwargs) - def get_symbol_suffix(self) -> str: - return '' + if self.is_found: + if self.needs_cblas: + # `name` may be 'blas' or 'blas64'; CBLAS library naming should be consistent + # with BLAS library naming, so just prepend 'c' and try to detect it. + try: + cblas_pc = PkgConfigDependency('c'+name, env, kwargs) + if cblas_pc.found(): + self.link_args += cblas_pc.link_args + self.compile_args += cblas_pc.compile_args + except DependencyException: + pass + + if not self.probe_symbols(self.link_args): + self.is_found = False + + +class NetlibBLASSystemDependency(BLASLAPACKMixin, NetlibMixin, SystemDependency): + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + # First, look for paths specified in a machine file + props = self.env.properties[self.for_machine].properties + if any(x in props for x in ['blas_includedir', 'blas_librarydir']): + self.detect_blas_machine_file(props) + + # Then look in standard directories by attempting to link + if not self.is_found: + extra_libdirs: T.List[str] = [] + self.detect(extra_libdirs) + + if self.is_found: + self.version = 'unknown' # no way to derive this from standard headers + + def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[T.List[str]] = None) -> None: + if lib_dirs is None: + lib_dirs = [] + if inc_dirs is None: + inc_dirs = [] + + if self.interface == 'lp64': + libnames = ['blas'] + cblas_headers = ['cblas.h'] + elif self.interface == 'ilp64': + libnames = ['blas64', 'blas'] + cblas_headers = ['cblas_64.h', 'cblas.h'] + + for libname in libnames: + link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) + if not link_arg: + continue + + # libblas may include CBLAS symbols (Debian builds it like this), + # but more often than not there's a separate libcblas library. Handle both cases. + if not self.probe_symbols(link_arg, check_cblas=False): + continue + if self.needs_cblas: + cblas_in_blas = self.probe_symbols(link_arg, check_cblas=True) + + if self.needs_cblas and not cblas_in_blas: + # We found libblas and it does not contain CBLAS symbols, so we need libcblas + cblas_libname = 'c' + libname + link_arg_cblas = self.clib_compiler.find_library(cblas_libname, self.env, lib_dirs) + if link_arg_cblas: + link_arg.extend(link_arg_cblas) + else: + # We didn't find CBLAS + continue + + self.is_found = True + self.link_args += link_arg + if self.needs_cblas: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in cblas_headers: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + # If we don't get here, we found the library but not the header - this may + # be okay, since projects may ship their own CBLAS header for portability) + self.compile_args += incdir_args + break + + def detect_blas_machine_file(self, props: dict) -> None: + has_dirs, _dirs = check_blas_machine_file('blas', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) + + +class NetlibLAPACKPkgConfigDependency(BLASLAPACKMixin, NetlibMixin, PkgConfigDependency): + def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + # TODO: add ILP64 (needs factory function like for OpenBLAS) + super().__init__(name, env, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + if self.is_found: + if self.needs_lapacke: + # Similar to CBLAS: there may be a separate liblapacke.so + try: + lapacke_pc = PkgConfigDependency(name+'e', env, kwargs) + if lapacke_pc.found(): + self.link_args += lapacke_pc.link_args + self.compile_args += lapacke_pc.compile_args + except DependencyException: + pass + + if not self.probe_symbols(self.link_args, lapack_only=True): + self.is_found = False + + +class NetlibLAPACKSystemDependency(BLASLAPACKMixin, NetlibMixin, SystemDependency): + def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T.Any]) -> None: + super().__init__(name, environment, kwargs) + self.feature_since = ('1.3.0', '') + self.parse_modules(kwargs) + + # First, look for paths specified in a machine file + props = self.env.properties[self.for_machine].properties + if any(x in props for x in ['lapack_includedir', 'lapack_librarydir']): + self.detect_lapack_machine_file(props) + + # Then look in standard directories by attempting to link + if not self.is_found: + extra_libdirs: T.List[str] = [] + self.detect(extra_libdirs) + + if self.is_found: + self.version = 'unknown' # no way to derive this from standard headers + + def detect(self, lib_dirs: T.Optional[T.List[str]] = None, inc_dirs: T.Optional[T.List[str]] = None) -> None: + if lib_dirs is None: + lib_dirs = [] + if inc_dirs is None: + inc_dirs = [] + + if self.interface == 'lp64': + libnames = ['lapack'] + lapacke_headers = ['lapacke.h'] + elif self.interface == 'ilp64': + libnames = ['lapack64', 'lapack'] + lapacke_headers = ['lapacke_64.h', 'lapacke.h'] + + for libname in libnames: + link_arg = self.clib_compiler.find_library(libname, self.env, lib_dirs) + if not link_arg: + continue + + if not self.probe_symbols(link_arg, check_lapacke=False, lapack_only=True): + continue + if self.needs_lapacke: + lapacke_in_lapack = self.probe_symbols(link_arg, check_lapacke=True, lapack_only=True) + + if self.needs_lapacke and not lapacke_in_lapack: + # We found liblapack and it does not contain LAPACKE symbols, so we need liblapacke + lapacke_libname = libname + 'e' + link_arg_lapacke = self.clib_compiler.find_library(lapacke_libname, self.env, lib_dirs) + if link_arg_lapacke: + link_arg.extend(link_arg_lapacke) + else: + # We didn't find LAPACKE + continue + + self.is_found = True + self.link_args += link_arg + if self.needs_lapacke: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + for hdr in lapacke_headers: + found_header, _ = self.clib_compiler.has_header(hdr, '', self.env, dependencies=[self], + extra_args=incdir_args) + if found_header: + # If we don't get here, we found the library but not the header - this may + # be okay, since projects may ship their own LAPACKE header for portability) + self.compile_args += incdir_args + break + + def detect_lapack_machine_file(self, props: dict) -> None: + has_dirs, _dirs = check_blas_machine_file('lapack', props) + if has_dirs: + libdir, incdir = _dirs + self.detect([libdir], [incdir]) + class AccelerateSystemDependency(BLASLAPACKMixin, SystemDependency): @@ -683,9 +899,10 @@ def detect_sdl(self) -> None: mlog.warning(f'MKLROOT env var set to {mklroot}, but not pointing to an MKL install') link_arg = self.clib_compiler.find_library('mkl_rt', self.env, lib_dirs) - incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] - found_header, _ = self.clib_compiler.has_header('mkl_version.h', '', self.env, - dependencies=[self], extra_args=incdir_args) + if link_arg: + incdir_args = [f'-I{inc_dir}' for inc_dir in inc_dirs] + found_header, _ = self.clib_compiler.has_header('mkl_version.h', '', self.env, + dependencies=[self], extra_args=incdir_args) if link_arg and found_header: self.is_found = True self.compile_args += incdir_args @@ -734,11 +951,18 @@ def openblas_factory(env: 'Environment', for_machine: 'MachineChoice', packages['openblas'] = openblas_factory -packages['netlib-blas'] = netlib_factory = DependencyFactory( - 'netlib-blas', - [DependencyMethods.PKGCONFIG], #, DependencyMethods.SYSTEM], - #system_class=NetlibSystemDependency, - pkgconfig_class=NetlibPkgConfigDependency, +packages['blas'] = netlib_factory = DependencyFactory( + 'blas', + [DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM], + pkgconfig_class=NetlibBLASPkgConfigDependency, + system_class=NetlibBLASSystemDependency, +) + +packages['lapack'] = netlib_factory = DependencyFactory( + 'lapack', + [DependencyMethods.PKGCONFIG, DependencyMethods.SYSTEM], + pkgconfig_class=NetlibLAPACKPkgConfigDependency, + system_class=NetlibLAPACKSystemDependency, ) From c1d25c8f68aa4a7d65f637a82bfdd95316825615 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Fri, 22 Dec 2023 10:48:08 +0100 Subject: [PATCH 23/25] BUG: fix macOS version checks so things work in very old versions The two things fixed here: - `xcrun` isn't available on very old versions (MacPorts still supports 10.5/10.6) - Version comparison is done correctly now with `mesonlib.version_compare` See https://github.com/numpy/numpy/issues/25406 for the bug report. --- mesonbuild/dependencies/blas_lapack.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index 92c24725a054..3bd4ae8425bd 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -742,12 +742,15 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. self.detect(kwargs) def check_macOS_recent_enough(self) -> bool: - # We need the SDK to be >=13.3 (meaning at least XCode 14.3) - cmd = ['xcrun', '-sdk', 'macosx', '--show-sdk-version'] - sdk_version = subprocess.run(cmd, capture_output=True, check=True, text=True).stdout.strip() macos_version = platform.mac_ver()[0] deploy_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', macos_version) - return sdk_version >= '13.3' and deploy_target >= '13.3' + if not mesonlib.version_compare(deploy_target, '>=13.3'): + return False + + # We also need the SDK to be >=13.3 (meaning at least XCode 14.3) + cmd = ['xcrun', '-sdk', 'macosx', '--show-sdk-version'] + sdk_version = subprocess.run(cmd, capture_output=True, check=True, text=True).stdout.strip() + return mesonlib.version_compare(sdk_version, '>=13.3') def detect(self, kwargs: T.Dict[str, T.Any]) -> None: from .framework import ExtraFrameworkDependency From 8650ab36d572d2df6665b152e371116a40697e4b Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Fri, 1 Aug 2025 09:46:41 +1000 Subject: [PATCH 24/25] fix long-standing linting problems --- mesonbuild/dependencies/__init__.py | 1 - mesonbuild/dependencies/blas_lapack.py | 24 ++++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mesonbuild/dependencies/__init__.py b/mesonbuild/dependencies/__init__.py index 196d990c63ec..aaa7167ea8e2 100644 --- a/mesonbuild/dependencies/__init__.py +++ b/mesonbuild/dependencies/__init__.py @@ -7,7 +7,6 @@ ExternalLibrary, DependencyException, DependencyMethods, BuiltinDependency, SystemDependency, get_leaf_external_dependencies) from .detect import find_external_dependency, get_dep_identifier, packages, _packages_accept_language -from .blas_lapack import openblas_factory __all__ = [ diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index 3bd4ae8425bd..af0c1310ac9d 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -25,7 +25,7 @@ from .. import mesonlib from ..mesonlib import MachineChoice, OptionKey -from .base import DependencyMethods, SystemDependency +from .base import DependencyMethods, SystemDependency, DependencyException from .cmake import CMakeDependency from .detect import packages from .factory import DependencyFactory, factory_methods @@ -33,6 +33,7 @@ if T.TYPE_CHECKING: from ..environment import Environment + from . factory import DependencyGenerator """ TODO: how to select BLAS interface layer (LP64, ILP64)? @@ -302,7 +303,7 @@ """ -def check_blas_machine_file(self, name: str, props: dict) -> T.Tuple[bool, T.List[str]]: +def check_blas_machine_file(name: str, props: dict) -> T.Tuple[bool, T.List[str]]: # TBD: do we need to support multiple extra dirs? incdir = props.get(f'{name}_includedir') assert incdir is None or isinstance(incdir, str) @@ -363,11 +364,11 @@ def check_symbols(self, compile_args, suffix=None, check_cblas=True, prototypes = "".join(f"void {symbol}{suffix}();\n" for symbol in symbols) calls = " ".join(f"{symbol}{suffix}();\n" for symbol in symbols) code = (f"{prototypes}" - "int main(int argc, const char *argv[])\n" - "{\n" + "int main(int argc, const char *argv[])\n" + "{\n" f" {calls}" - " return 0;\n" - "}" + " return 0;\n" + "}" ) code = '''#ifdef __cplusplus extern "C" { @@ -719,7 +720,6 @@ def detect_lapack_machine_file(self, props: dict) -> None: self.detect([libdir], [incdir]) - class AccelerateSystemDependency(BLASLAPACKMixin, SystemDependency): """ Accelerate is always installed on macOS, and not available on other OSes. @@ -765,7 +765,6 @@ def detect(self, kwargs: T.Dict[str, T.Any]) -> None: # We won't check symbols here, because Accelerate is built in a consistent fashion # with known symbol mangling, unlike OpenBLAS or Netlib BLAS/LAPACK. - return None def get_symbol_suffix(self) -> str: return '$NEWLAPACK' if self.interface == 'lp64' else '$NEWLAPACK$ILP64' @@ -788,7 +787,7 @@ def parse_mkl_options(self, kwargs: T.Dict[str, T.Any]) -> None: if not threading_module: self.threading = 'iomp' elif len(threading_module) > 1: - raise mesonlib.MesonException(f'Multiple threading arguments: {threading_modules}') + raise mesonlib.MesonException(f'Multiple threading arguments: {threading_module}') else: # We have a single threading option specified - validate and process it opt = threading_module[0] @@ -802,7 +801,7 @@ def parse_mkl_options(self, kwargs: T.Dict[str, T.Any]) -> None: if not sdl_module: self.use_sdl = 'auto' elif len(sdl_module) > 1: - raise mesonlib.MesonException(f'Multiple sdl arguments: {threading_modules}') + raise mesonlib.MesonException(f'Multiple sdl arguments: {threading_module}') else: # We have a single sdl option specified - validate and process it opt = sdl_module[0] @@ -829,8 +828,6 @@ def parse_mkl_options(self, kwargs: T.Dict[str, T.Any]) -> None: raise mesonlib.MesonException(f'Linking SDL implies using LP64 and Intel OpenMP, found ' f'conflicting options: {self.interface}, {self.threading}') - return None - class MKLPkgConfigDependency(BLASLAPACKMixin, MKLMixin, PkgConfigDependency): """ @@ -880,7 +877,6 @@ def __init__(self, name: str, environment: 'Environment', kwargs: T.Dict[str, T. if self.use_sdl: self.detect_sdl() - return None def detect_sdl(self) -> None: # Use MKLROOT in addition to standard libdir(s) @@ -910,7 +906,7 @@ def detect_sdl(self) -> None: self.is_found = True self.compile_args += incdir_args self.link_args += link_arg - if not sys.platform == 'win32': + if sys.platform != 'win32': self.link_args += ['-lpthread', '-lm', '-ldl'] # Determine MKL version From b45087ff5f65ff30ee540623aab36c7e51f4d315 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Fri, 1 Aug 2025 10:43:44 +1000 Subject: [PATCH 25/25] update syntax for MKL get_option() --- mesonbuild/dependencies/blas_lapack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesonbuild/dependencies/blas_lapack.py b/mesonbuild/dependencies/blas_lapack.py index af0c1310ac9d..9fa7d35b05a1 100644 --- a/mesonbuild/dependencies/blas_lapack.py +++ b/mesonbuild/dependencies/blas_lapack.py @@ -855,7 +855,7 @@ def __init__(self, name: str, env: 'Environment', kwargs: T.Dict[str, T.Any]) -> # available before the .pc file for SDL self.use_sdl = False - static_opt = kwargs.get('static', env.coredata.get_option(OptionKey('prefer_static'))) + static_opt = kwargs.get('static', env.coredata.optstore.get_value_for(OptionKey('prefer_static'))) libtype = 'static' if static_opt else 'dynamic' if self.use_sdl: