# ------------------------------------------------------------------ # Copyright (c) 2020 PyInstaller Development Team. # # This file is distributed under the terms of the GNU General Public # License (version 2.0 or later). # # The full license is available in LICENSE, distributed with # this software. # # SPDX-License-Identifier: GPL-2.0-or-later # ------------------------------------------------------------------ from _pyinstaller_hooks_contrib.compat import importlib_metadata from packaging.version import Version from PyInstaller.compat import is_linux from PyInstaller.utils.hooks import ( collect_data_files, collect_dynamic_libs, collect_submodules, get_module_attribute, is_module_satisfies, logger, ) # Determine the name of `tensorflow` dist; this is available under different names (releases vs. nightly, plus build # variants). We need to determine the dist that we are dealing with, so we can query its version and metadata. _CANDIDATE_DIST_NAMES = ( "tensorflow", "tensorflow-cpu", "tensorflow-gpu", "tensorflow-intel", "tensorflow-rocm", "tensorflow-macos", "tensorflow-aarch64", "tensorflow-cpu-aws", "tf-nightly", "tf-nightly-cpu", "tf-nightly-gpu", "tf-nightly-rocm", "intel-tensorflow", "intel-tensorflow-avx512", ) dist = None for candidate_dist_name in _CANDIDATE_DIST_NAMES: try: dist = importlib_metadata.distribution(candidate_dist_name) break except importlib_metadata.PackageNotFoundError: continue version = None if dist is None: logger.warning( "hook-tensorflow: failed to determine tensorflow dist name! Reading version from tensorflow.__version__!" ) try: version = get_module_attribute("tensorflow", "__version__") except Exception as e: raise Exception("Failed to read tensorflow.__version__") from e else: logger.info("hook-tensorflow: tensorflow dist name: %s", dist.name) version = dist.version # Parse version logger.info("hook-tensorflow: tensorflow version: %s", version) try: version = Version(version) except Exception as e: raise Exception("Failed to parse tensorflow version!") from e # Exclude from data collection: # - development headers in include subdirectory # - XLA AOT runtime sources # - libtensorflow_framework and libtensorflow_cc (since TF 2.12) shared libraries (to avoid duplication) # - import library (.lib) files (Windows-only) data_excludes = [ "include", "xla_aot_runtime_src", "libtensorflow_framework.*", "libtensorflow_cc.*", "**/*.lib", ] # Under tensorflow 2.3.0 (the most recent version at the time of writing), _pywrap_tensorflow_internal extension module # ends up duplicated; once as an extension, and once as a shared library. In addition to increasing program size, this # also causes problems on macOS, so we try to prevent the extension module "variant" from being picked up. # # See pyinstaller/pyinstaller-hooks-contrib#49 for details. # # With PyInstaller >= 6.0, this issue is alleviated, because the binary dependency analysis (which picks up the # extension in question as a shared library that other extensions are linked against) now preserves the parent directory # layout, and creates a symbolic link to the top-level application directory. if is_module_satisfies('PyInstaller >= 6.0'): excluded_submodules = [] else: excluded_submodules = ['tensorflow.python._pywrap_tensorflow_internal'] def _submodules_filter(x): return x not in excluded_submodules if version < Version("1.15.0a0"): # 1.14.x and earlier: collect everything from tensorflow hiddenimports = collect_submodules('tensorflow', filter=_submodules_filter) datas = collect_data_files('tensorflow', excludes=data_excludes) elif version >= Version("1.15.0a0") and version < Version("2.2.0a0"): # 1.15.x - 2.1.x: collect everything from tensorflow_core hiddenimports = collect_submodules('tensorflow_core', filter=_submodules_filter) datas = collect_data_files('tensorflow_core', excludes=data_excludes) # Under 1.15.x, we seem to fail collecting a specific submodule, and need to add it manually... if version < Version("2.0.0a0"): hiddenimports += ['tensorflow_core._api.v1.compat.v2.summary.experimental'] else: # 2.2.0 and newer: collect everything from tensorflow again hiddenimports = collect_submodules('tensorflow', filter=_submodules_filter) datas = collect_data_files('tensorflow', excludes=data_excludes) # From 2.6.0 on, we also need to explicitly collect keras (due to lazy mapping of tensorflow.keras.xyz -> keras.xyz) if version >= Version("2.6.0a0"): hiddenimports += collect_submodules('keras') # Starting with 2.14.0, we need `ml_dtypes` among hidden imports. if version >= Version("2.14.0"): hiddenimports += ['ml_dtypes'] binaries = [] excludedimports = excluded_submodules # Suppress warnings for missing hidden imports generated by this hook. # Requires PyInstaller > 5.1 (with pyinstaller/pyinstaller#6914 merged); no-op otherwise. warn_on_missing_hiddenimports = False # Collect the AutoGraph part of `tensorflow` code, to avoid a run-time warning about AutoGraph being unavailable: # `WARNING:tensorflow:AutoGraph is not available in this environment: functions lack code information. ...` # The warning is emitted if source for `log` function from `tensorflow.python.autograph.utils.ag_logging` cannot be # looked up. Not sure if we need sources for other parts of `tesnorflow`, though. # Requires PyInstaller >= 5.3, no-op in older versions. module_collection_mode = { 'tensorflow.python.autograph': 'py+pyz', } # Linux builds of tensorflow can optionally use CUDA from nvidia-* packages. If we managed to obtain dist, query the # requirements from metadata (the `and-cuda` extra marker), and convert them to module names. # # NOTE: while the installation of nvidia-* packages via `and-cuda` extra marker is not gated by the OS version check, # it is effectively available only on Linux (last Windows-native build that supported GPU is v2.10.0, and assumed that # CUDA is externally available). if is_linux and dist is not None: def _infer_nvidia_hiddenimports(): import packaging.requirements from _pyinstaller_hooks_contrib.utils import nvidia_cuda as cudautils requirements = [packaging.requirements.Requirement(req) for req in dist.requires or []] env = {'extra': 'and-cuda'} requirements = [req.name for req in requirements if req.marker is None or req.marker.evaluate(env)] return cudautils.infer_hiddenimports_from_requirements(requirements) try: nvidia_hiddenimports = _infer_nvidia_hiddenimports() except Exception: # Log the exception, but make it non-fatal logger.warning("hook-tensorflow: failed to infer NVIDIA CUDA hidden imports!", exc_info=True) nvidia_hiddenimports = [] logger.info("hook-tensorflow: inferred hidden imports for CUDA libraries: %r", nvidia_hiddenimports) hiddenimports += nvidia_hiddenimports # Collect the tensorflow-plugins (pluggable device plugins) hiddenimports += ['tensorflow-plugins'] binaries += collect_dynamic_libs('tensorflow-plugins') # On Linux, prevent binary dependency analysis from generating symbolic links for libtensorflow_cc.so.2, # libtensorflow_framework.so.2, and _pywrap_tensorflow_internal.so to the top-level application directory. These # symbolic links seem to confuse tensorflow about its location (likely because code in one of the libraries looks up the # library file's location, but does not fully resolve it), which in turn prevents it from finding the collected CUDA # libraries in the nvidia/cu* package directories. # # The `bindepend_symlink_suppression` hook attribute requires PyInstaller >= 6.11, and is no-op in earlier versions. if is_linux: bindepend_symlink_suppression = [ '**/libtensorflow_cc.so*', '**/libtensorflow_framework.so*', '**/_pywrap_tensorflow_internal.so', ]