1 From 019ed04331695bb6f5c5fff70dfced34c4ba9012 Mon Sep 17 00:00:00 2001
2 From: Daniel Mensinger <daniel@mensinger-ka.de>
3 Date: Thu, 16 Jul 2020 20:29:34 +0200
4 Subject: [PATCH 1/3] mdata: Generate mesondata.py from */data folders
7 mesonbuild/mesondata.py | 374 ++++++++++++++++++++++++++++++++++++++++
8 tools/gen_data.py | 139 +++++++++++++++
9 2 files changed, 513 insertions(+)
10 create mode 100644 mesonbuild/mesondata.py
11 create mode 100755 tools/gen_data.py
13 diff --git a/mesonbuild/mesondata.py b/mesonbuild/mesondata.py
15 index 0000000000..1f223c251b
17 +++ b/mesonbuild/mesondata.py
19 +# Copyright 2020 The Meson development team
21 +# Licensed under the Apache License, Version 2.0 (the "License");
22 +# you may not use this file except in compliance with the License.
23 +# You may obtain a copy of the License at
25 +# http://www.apache.org/licenses/LICENSE-2.0
27 +# Unless required by applicable law or agreed to in writing, software
28 +# distributed under the License is distributed on an "AS IS" BASIS,
29 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
30 +# See the License for the specific language governing permissions and
31 +# limitations under the License.
35 +#### WARNING: This is an automatically generated file! Do not edit!
36 +#### Generated by tools/gen_data.py
40 +from pathlib import Path
44 + from .environment import Environment
46 +######################
47 +# BEGIN Data section #
48 +######################
50 +file_0_data_CMakeListsLLVM_txt = '''\
51 +cmake_minimum_required(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION} )
53 +set(PACKAGE_FOUND FALSE)
56 + find_package(LLVM REQUIRED CONFIG QUIET)
58 + # ARCHS has to be set via the CMD interface
59 + if(LLVM_FOUND OR "${ARCHS}" STREQUAL "")
63 + list(GET ARCHS 0 CMAKE_LIBRARY_ARCHITECTURE)
64 + list(REMOVE_AT ARCHS 0)
68 + set(PACKAGE_FOUND TRUE)
70 + foreach(mod IN LISTS LLVM_MESON_MODULES)
75 + # Generate a lower and upper case version
76 + string(TOLOWER "${mod}" mod_L)
77 + string(TOUPPER "${mod}" mod_U)
79 + # Get the mapped components
80 + llvm_map_components_to_libnames(out_mods ${mod} ${mod_L} ${mod_U})
82 + list(REMOVE_DUPLICATES out_mods)
84 + # Make sure that the modules exist
85 + foreach(i IN LISTS out_mods)
87 + list(APPEND real_mods ${i})
91 + # Set the output variables
92 + set(MESON_LLVM_TARGETS_${mod} ${real_mods})
93 + foreach(i IN LISTS real_mods)
94 + set(MESON_TARGET_TO_LLVM_${i} ${mod})
98 + # Check the following variables:
99 + # LLVM_PACKAGE_VERSION
101 + # LLVM_VERSION_STRING
102 + if(NOT DEFINED PACKAGE_VERSION)
103 + if(DEFINED LLVM_PACKAGE_VERSION)
104 + set(PACKAGE_VERSION "${LLVM_PACKAGE_VERSION}")
105 + elseif(DEFINED LLVM_VERSION)
106 + set(PACKAGE_VERSION "${LLVM_VERSION}")
107 + elseif(DEFINED LLVM_VERSION_STRING)
108 + set(PACKAGE_VERSION "${LLVM_VERSION_STRING}")
112 + # Check the following variables:
116 + if(DEFINED LLVM_LIBRARIES)
117 + set(libs LLVM_LIBRARIES)
118 + elseif(DEFINED LLVM_LIBS)
119 + set(libs LLVM_LIBS)
122 + # Check the following variables:
123 + # LLVM_INCLUDE_DIRS
127 + if(DEFINED LLVM_INCLUDE_DIRS)
128 + set(includes LLVM_INCLUDE_DIRS)
129 + elseif(DEFINED LLVM_INCLUDES)
130 + set(includes LLVM_INCLUDES)
131 + elseif(DEFINED LLVM_INCLUDE_DIR)
132 + set(includes LLVM_INCLUDE_DIR)
135 + # Check the following variables:
138 + if(DEFINED LLVM_DEFINITIONS)
139 + set(definitions LLVM_DEFINITIONS)
142 + set(PACKAGE_INCLUDE_DIRS "${${includes}}")
143 + set(PACKAGE_DEFINITIONS "${${definitions}}")
144 + set(PACKAGE_LIBRARIES "${${libs}}")
148 +file_1_data_CMakePathInfo_txt = '''\
149 +cmake_minimum_required(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}.${CMAKE_PATCH_VERSION})
152 +list(APPEND TMP_PATHS_LIST ${CMAKE_PREFIX_PATH})
153 +list(APPEND TMP_PATHS_LIST ${CMAKE_FRAMEWORK_PATH})
154 +list(APPEND TMP_PATHS_LIST ${CMAKE_APPBUNDLE_PATH})
155 +list(APPEND TMP_PATHS_LIST $ENV{CMAKE_PREFIX_PATH})
156 +list(APPEND TMP_PATHS_LIST $ENV{CMAKE_FRAMEWORK_PATH})
157 +list(APPEND TMP_PATHS_LIST $ENV{CMAKE_APPBUNDLE_PATH})
158 +list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_PREFIX_PATH})
159 +list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_FRAMEWORK_PATH})
160 +list(APPEND TMP_PATHS_LIST ${CMAKE_SYSTEM_APPBUNDLE_PATH})
163 +if(CMAKE_LIBRARY_ARCHITECTURE_REGEX)
164 + file(GLOB implicit_dirs RELATIVE /lib /lib/*-linux-gnu* )
165 + foreach(dir ${implicit_dirs})
166 + if("${dir}" MATCHES "${CMAKE_LIBRARY_ARCHITECTURE_REGEX}")
167 + list(APPEND LIB_ARCH_LIST "${dir}")
172 +# "Export" these variables:
173 +set(MESON_ARCH_LIST ${LIB_ARCH_LIST})
174 +set(MESON_PATHS_LIST ${TMP_PATHS_LIST})
175 +set(MESON_CMAKE_ROOT ${CMAKE_ROOT})
176 +set(MESON_CMAKE_SYSROOT ${CMAKE_SYSROOT})
177 +set(MESON_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH})
179 +message(STATUS ${TMP_PATHS_LIST})
182 +file_2_data_CMakeLists_txt = '''\
183 +# fail noisily if attempt to use this file without setting:
184 +# cmake_minimum_required(VERSION ${CMAKE_VERSION})
185 +# project(... LANGUAGES ...)
187 +cmake_policy(SET CMP0000 NEW)
189 +set(PACKAGE_FOUND FALSE)
190 +set(_packageName "${NAME}")
191 +string(TOUPPER "${_packageName}" PACKAGE_NAME)
194 + find_package("${NAME}" QUIET COMPONENTS ${COMPS})
196 + # ARCHS has to be set via the CMD interface
197 + if(${_packageName}_FOUND OR ${PACKAGE_NAME}_FOUND OR "${ARCHS}" STREQUAL "")
201 + list(GET ARCHS 0 CMAKE_LIBRARY_ARCHITECTURE)
202 + list(REMOVE_AT ARCHS 0)
205 +if(${_packageName}_FOUND OR ${PACKAGE_NAME}_FOUND)
206 + set(PACKAGE_FOUND TRUE)
208 + # Check the following variables:
211 + # FOO_VERSION_STRING
212 + # Foo_VERSION_STRING
213 + if(NOT DEFINED PACKAGE_VERSION)
214 + if(DEFINED ${_packageName}_VERSION)
215 + set(PACKAGE_VERSION "${${_packageName}_VERSION}")
216 + elseif(DEFINED ${PACKAGE_NAME}_VERSION)
217 + set(PACKAGE_VERSION "${${PACKAGE_NAME}_VERSION}")
218 + elseif(DEFINED ${_packageName}_VERSION_STRING)
219 + set(PACKAGE_VERSION "${${_packageName}_VERSION_STRING}")
220 + elseif(DEFINED ${PACKAGE_NAME}_VERSION_STRING)
221 + set(PACKAGE_VERSION "${${PACKAGE_NAME}_VERSION_STRING}")
225 + # Check the following variables:
231 + if(DEFINED ${_packageName}_LIBRARIES)
232 + set(libs ${_packageName}_LIBRARIES)
233 + elseif(DEFINED ${PACKAGE_NAME}_LIBRARIES)
234 + set(libs ${PACKAGE_NAME}_LIBRARIES)
235 + elseif(DEFINED ${_packageName}_LIBS)
236 + set(libs ${_packageName}_LIBS)
237 + elseif(DEFINED ${PACKAGE_NAME}_LIBS)
238 + set(libs ${PACKAGE_NAME}_LIBS)
241 + # Check the following variables:
249 + if(DEFINED ${_packageName}_INCLUDE_DIRS)
250 + set(includes ${_packageName}_INCLUDE_DIRS)
251 + elseif(DEFINED ${PACKAGE_NAME}_INCLUDE_DIRS)
252 + set(includes ${PACKAGE_NAME}_INCLUDE_DIRS)
253 + elseif(DEFINED ${_packageName}_INCLUDES)
254 + set(includes ${_packageName}_INCLUDES)
255 + elseif(DEFINED ${PACKAGE_NAME}_INCLUDES)
256 + set(includes ${PACKAGE_NAME}_INCLUDES)
257 + elseif(DEFINED ${_packageName}_INCLUDE_DIR)
258 + set(includes ${_packageName}_INCLUDE_DIR)
259 + elseif(DEFINED ${PACKAGE_NAME}_INCLUDE_DIR)
260 + set(includes ${PACKAGE_NAME}_INCLUDE_DIR)
263 + # Check the following variables:
267 + if(DEFINED ${_packageName}_DEFINITIONS)
268 + set(definitions ${_packageName}_DEFINITIONS)
269 + elseif(DEFINED ${PACKAGE_NAME}_DEFINITIONS)
270 + set(definitions ${PACKAGE_NAME}_DEFINITIONS)
273 + set(PACKAGE_INCLUDE_DIRS "${${includes}}")
274 + set(PACKAGE_DEFINITIONS "${${definitions}}")
275 + set(PACKAGE_LIBRARIES "${${libs}}")
279 +file_3_data_preload_cmake = '''\
284 +set(MESON_PS_LOADED ON)
286 +# Dummy macros that have a special meaning in the meson code
287 +macro(meson_ps_execute_delayed_calls)
290 +macro(meson_ps_reload_vars)
293 +# Helper macro to inspect the current CMake state
294 +macro(meson_ps_inspect_vars)
295 + set(MESON_PS_CMAKE_CURRENT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
296 + set(MESON_PS_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
297 + meson_ps_execute_delayed_calls()
301 +# Override some system functions with custom code and forward the args
302 +# to the original function
303 +macro(add_custom_command)
304 + meson_ps_inspect_vars()
305 + _add_custom_command(${ARGV})
308 +macro(add_custom_target)
309 + meson_ps_inspect_vars()
310 + _add_custom_target(${ARGV})
314 + meson_ps_inspect_vars()
315 + _set_property(${ARGV})
318 +function(set_source_files_properties)
321 + set(PROPERTIES OFF)
325 + if("${ARGV${I}}" STREQUAL "PROPERTIES")
328 + list(APPEND FILES "${ARGV${I}}")
331 + math(EXPR I "${I} + 1")
334 + math(EXPR PROP_IDX "${ID_IDX} + 1")
336 + set(ID "${ARGV${ID_IDX}}")
337 + set(PROP "${ARGV${PROP_IDX}}")
339 + set_property(SOURCE ${FILES} PROPERTY "${ID}" "${PROP}")
340 + math(EXPR I "${I} + 2")
345 +set(MESON_PS_DELAYED_CALLS add_custom_command;add_custom_target;set_property)
346 +meson_ps_reload_vars()
350 +####################
351 +# END Data section #
352 +####################
355 + def __init__(self, path: Path, sha256sum: str, data: str) -> None:
357 + self.sha256sum = sha256sum
360 + def write_once(self, path: Path) -> None:
361 + if not path.exists():
362 + path.write_text(self.data)
364 + def write_to_private(self, env: 'Environment') -> Path:
365 + out_file = Path(env.scratch_dir) / 'data' / self.path.name
366 + out_file.parent.mkdir(exist_ok=True)
367 + self.write_once(out_file)
372 + 'dependencies/data/CMakeListsLLVM.txt': DataFile(
373 + Path('dependencies/data/CMakeListsLLVM.txt'),
374 + '412cec3315597041a978d018cdaca282dcd47693793540da88ae2f80d0cbd7cd',
375 + file_0_data_CMakeListsLLVM_txt,
377 + 'dependencies/data/CMakePathInfo.txt': DataFile(
378 + Path('dependencies/data/CMakePathInfo.txt'),
379 + '90da8b443982d9c87139b7dc84228eb58cab4315764949637208f25e2bda7db2',
380 + file_1_data_CMakePathInfo_txt,
382 + 'dependencies/data/CMakeLists.txt': DataFile(
383 + Path('dependencies/data/CMakeLists.txt'),
384 + '71a2d58381f912bbfb1c8709884d34d721f682edf2fca001e1f582f0bffd0da7',
385 + file_2_data_CMakeLists_txt,
387 + 'cmake/data/preload.cmake': DataFile(
388 + Path('cmake/data/preload.cmake'),
389 + '064d047b18a5c919ad016b838bed50c5d40aebe9e53da0e70eff9d52a2c1ca1f',
390 + file_3_data_preload_cmake,
393 diff --git a/tools/gen_data.py b/tools/gen_data.py
395 index 0000000000..2cc05a44e7
397 +++ b/tools/gen_data.py
399 +#!/usr/bin/env python3
401 +# Copyright 2020 Daniel Mensinger
403 +# Licensed under the Apache License, Version 2.0 (the "License");
404 +# you may not use this file except in compliance with the License.
405 +# You may obtain a copy of the License at
407 +# http://www.apache.org/licenses/LICENSE-2.0
409 +# Unless required by applicable law or agreed to in writing, software
410 +# distributed under the License is distributed on an "AS IS" BASIS,
411 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
412 +# See the License for the specific language governing permissions and
413 +# limitations under the License.
419 +from pathlib import Path
420 +from datetime import datetime
426 + def __init__(self, path: Path, root: Path):
428 + self.id = self.path.relative_to(root)
429 + self.data_str = f'file_{DataFile.file_counter}_data_' + re.sub('[^a-zA-Z0-9]', '_', self.path.name)
430 + DataFile.file_counter += 1
432 + b = self.path.read_bytes()
433 + self.data = b.decode()
434 + self.sha256sum = hashlib.sha256(b).hexdigest()
436 + def __repr__(self) -> str:
437 + return f'<{type(self).__name__}: [{self.sha256sum}] {self.id}>'
440 + root_dir = Path(__file__).resolve().parents[1]
441 + mesonbuild_dir = root_dir / 'mesonbuild'
442 + out_file = mesonbuild_dir / 'mesondata.py'
444 + data_dirs = mesonbuild_dir.glob('**/data')
446 + data_files: T.List[DataFile] = []
448 + for d in data_dirs:
449 + for p in d.iterdir():
450 + data_files += [DataFile(p, mesonbuild_dir)]
452 + print(f'Found {len(data_files)} data files')
454 + # Generate the data script
457 + data += textwrap.dedent(f'''\
458 + # Copyright {datetime.today().year} The Meson development team
460 + # Licensed under the Apache License, Version 2.0 (the "License");
461 + # you may not use this file except in compliance with the License.
462 + # You may obtain a copy of the License at
464 + # http://www.apache.org/licenses/LICENSE-2.0
466 + # Unless required by applicable law or agreed to in writing, software
467 + # distributed under the License is distributed on an "AS IS" BASIS,
468 + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
469 + # See the License for the specific language governing permissions and
470 + # limitations under the License.
474 + #### WARNING: This is an automatically generated file! Do not edit!
475 + #### Generated by {Path(__file__).resolve().relative_to(root_dir)}
479 + from pathlib import Path
482 + if T.TYPE_CHECKING:
483 + from .environment import Environment
485 + ######################
486 + # BEGIN Data section #
487 + ######################
491 + for i in data_files:
492 + data += f"{i.data_str} = '''\\\n{i.data}'''\n\n"
494 + data += textwrap.dedent(f'''
495 + ####################
496 + # END Data section #
497 + ####################
500 + def __init__(self, path: Path, sha256sum: str, data: str) -> None:
502 + self.sha256sum = sha256sum
505 + def write_once(self, path: Path) -> None:
506 + if not path.exists():
507 + path.write_text(self.data)
509 + def write_to_private(self, env: 'Environment') -> Path:
510 + out_file = Path(env.scratch_dir) / 'data' / self.path.name
511 + out_file.parent.mkdir(exist_ok=True)
512 + self.write_once(out_file)
519 + for i in data_files:
520 + data += textwrap.indent(textwrap.dedent(f"""\
521 + '{i.id}': DataFile(
528 + data += textwrap.dedent('''\
532 + print(f'Updating {out_file}')
533 + out_file.write_text(data)
536 +if __name__ == '__main__':
539 From 05ddd6543d4c4fc33b4c64f26291e73f49733f71 Mon Sep 17 00:00:00 2001
540 From: Daniel Mensinger <daniel@mensinger-ka.de>
541 Date: Thu, 16 Jul 2020 20:33:57 +0200
542 Subject: [PATCH 2/3] mdata: remove setuptools and use mesondata instead
545 mesonbuild/cmake/interpreter.py | 5 ++---
546 mesonbuild/dependencies/base.py | 6 ++----
548 4 files changed, 4 insertions(+), 16 deletions(-)
550 diff --git a/mesonbuild/cmake/interpreter.py b/mesonbuild/cmake/interpreter.py
551 index 05169478e8..f404109cf6 100644
552 --- a/mesonbuild/cmake/interpreter.py
553 +++ b/mesonbuild/cmake/interpreter.py
555 # This class contains the basic functionality needed to run any interpreter
556 # or an interpreter-based tool.
558 -import pkg_resources
560 from .common import CMakeException, CMakeTarget, TargetOptions
561 from .client import CMakeClient, RequestCMakeInputs, RequestConfigure, RequestCompute, RequestCodeModel
562 from .fileapi import CMakeFileAPI
564 from .. import mlog, mesonlib
565 from ..environment import Environment
566 from ..mesonlib import MachineChoice, OrderedSet, version_compare
567 +from ..mesondata import mesondata
568 from ..compilers.compilers import lang_suffixes, header_suffixes, obj_suffixes, lib_suffixes, is_header
569 from enum import Enum
570 from functools import lru_cache
571 @@ -814,7 +813,7 @@ def configure(self, extra_cmake_options: T.List[str]) -> None:
572 raise CMakeException('Unable to find CMake')
573 self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True)
575 - preload_file = pkg_resources.resource_filename('mesonbuild', 'cmake/data/preload.cmake')
576 + preload_file = mesondata['cmake/data/preload.cmake'].write_to_private(self.env)
578 # Prefere CMAKE_PROJECT_INCLUDE over CMAKE_TOOLCHAIN_FILE if possible,
579 # since CMAKE_PROJECT_INCLUDE was actually designed for code injection.
580 diff --git a/mesonbuild/dependencies/base.py b/mesonbuild/dependencies/base.py
581 index 23701da957..4c9c9fe18a 100644
582 --- a/mesonbuild/dependencies/base.py
583 +++ b/mesonbuild/dependencies/base.py
585 from enum import Enum
586 from pathlib import Path, PurePath
588 -import pkg_resources
591 from .. import mesonlib
592 from ..compilers import clib_langs
594 from ..mesonlib import MachineChoice, MesonException, OrderedSet, PerMachine
595 from ..mesonlib import Popen_safe, version_compare_many, version_compare, listify, stringlistify, extract_as_list, split_args
596 from ..mesonlib import Version, LibType
597 +from ..mesondata import mesondata
600 from ..compilers.compilers import CompilerType # noqa: F401
601 @@ -1512,8 +1511,7 @@ def _setup_cmake_dir(self, cmake_file: str) -> str:
602 build_dir = self._get_build_dir()
604 # Insert language parameters into the CMakeLists.txt and write new CMakeLists.txt
605 - # Per the warning in pkg_resources, this is *not* a path and os.path and Pathlib are *not* safe to use here.
606 - cmake_txt = pkg_resources.resource_string('mesonbuild', 'dependencies/data/' + cmake_file).decode()
607 + cmake_txt = mesondata['dependencies/data/' + cmake_file].data
609 # In general, some Fortran CMake find_package() also require C language enabled,
610 # even if nothing from C is directly used. An easy Fortran example that fails
611 diff --git a/setup.py b/setup.py
612 index 1f95be70c8..145f19c522 100644
616 'mesonbuild.scripts',
617 'mesonbuild.templates',
620 - 'mesonbuild.dependencies': ['data/CMakeLists.txt', 'data/CMakeListsLLVM.txt', 'data/CMakePathInfo.txt'],
621 - 'mesonbuild.cmake': ['data/run_ctgt.py', 'data/preload.cmake'],
624 if sys.platform != 'win32':
625 # Only useful on UNIX-like systems
630 - package_data=package_data,
631 entry_points=entries,
632 data_files=data_files,)
634 From 393d6e133d9abd584a2fc414971628e84ea48b7c Mon Sep 17 00:00:00 2001
635 From: Daniel Mensinger <daniel@mensinger-ka.de>
636 Date: Thu, 16 Jul 2020 20:34:15 +0200
637 Subject: [PATCH 3/3] mdata: Add test to ensure mesondata.py is up-to-date
640 run_unittests.py | 32 ++++++++++++++++++++++++++++++++
641 1 file changed, 32 insertions(+)
643 diff --git a/run_unittests.py b/run_unittests.py
644 index 820b705b54..2c03a3e75c 100755
645 --- a/run_unittests.py
646 +++ b/run_unittests.py
647 @@ -1485,6 +1485,38 @@ def test_all_functions_defined_in_ast_interpreter(self):
648 astint = AstInterpreter('.', '', '')
649 self.assertEqual(set(interp.funcs.keys()), set(astint.funcs.keys()))
651 + def test_mesondata_is_up_to_date(self):
652 + from mesonbuild.mesondata import mesondata
653 + err_msg = textwrap.dedent('''
655 + ###########################################################
656 + ### mesonbuild.mesondata is not up-to-date ###
657 + ### Please regenerate it by running tools/gen_data.py ###
658 + ###########################################################
662 + root_dir = Path(__file__).resolve().parent
663 + mesonbuild_dir = root_dir / 'mesonbuild'
665 + data_dirs = mesonbuild_dir.glob('**/data')
666 + data_files = [] # type: T.List[T.Tuple(str, str)]
668 + for i in data_dirs:
669 + for p in i.iterdir():
670 + data_files += [(p.relative_to(mesonbuild_dir).as_posix(), hashlib.sha256(p.read_bytes()).hexdigest())]
672 + from pprint import pprint
673 + current_files = set(mesondata.keys())
674 + scanned_files = set([x[0] for x in data_files])
676 + self.assertSetEqual(current_files, scanned_files, err_msg + 'Data files were added or removed\n')
678 + for i in data_files:
679 + if mesondata[i[0]].sha256sum != i[1]:
682 + self.assertListEqual(errors, [], err_msg + 'Files were changed')
684 class BasePlatformTests(unittest.TestCase):