gst-env.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import contextlib
   5 import glob
   6 import json
   7 import os
   8 import platform
   9 import re
  10 import site
  11 import shlex
  12 import shutil
  13 import subprocess
  14 import sys
  15 import tempfile
  16 import pathlib
  17 import signal
  18 from pathlib import PurePath
  19
  20 from distutils.sysconfig import get_python_lib
  21 from distutils.util import strtobool
  22
  23 from scripts.common import get_meson
  24 from scripts.common import git
  25 from scripts.common import win32_get_short_path_name
  26 from scripts.common import get_wine_shortpath
  27
  28 SCRIPTDIR = os.path.dirname(os.path.realpath(__file__))
  29 PREFIX_DIR = os.path.join(SCRIPTDIR, 'prefix')
  30 # Look for the following build dirs: `build` `_build` `builddir`
  31 DEFAULT_BUILDDIR = os.path.join(SCRIPTDIR, 'build')
  32 if not os.path.exists(DEFAULT_BUILDDIR):
  33     DEFAULT_BUILDDIR = os.path.join(SCRIPTDIR, '_build')
  34 if not os.path.exists(DEFAULT_BUILDDIR):
  35     DEFAULT_BUILDDIR = os.path.join(SCRIPTDIR, 'builddir')
  36
  37 TYPELIB_REG = re.compile(r'.*\.typelib$')
  38 SHAREDLIB_REG = re.compile(r'\.so|\.dylib|\.dll')
  39
  40 # libdir is expanded from option of the same name listed in the `meson
  41 # introspect --buildoptions` output.
  42 GSTPLUGIN_FILEPATH_REG_TEMPLATE = r'.*/{libdir}/gstreamer-1.0/[^/]+$'
  43 GSTPLUGIN_FILEPATH_REG = None
  44
  45 def listify(o):
  46     if isinstance(o, str):
  47         return [o]
  48     if isinstance(o, list):
  49         return o
  50     raise AssertionError('Object {!r} must be a string or a list'.format(o))
  51
  52 def stringify(o):
  53     if isinstance(o, str):
  54         return o
  55     if isinstance(o, list):
  56         if len(o) == 1:
  57             return o[0]
  58         raise AssertionError('Did not expect object {!r} to have more than one element'.format(o))
  59     raise AssertionError('Object {!r} must be a string or a list'.format(o))
  60
  61 def prepend_env_var(env, var, value, sysroot):
  62     if value.startswith(sysroot):
  63         value = value[len(sysroot):]
  64     # Try not to exceed maximum length limits for env vars on Windows
  65     if os.name == 'nt':
  66         value = win32_get_short_path_name(value)
  67     env_val = env.get(var, '')
  68     val = os.pathsep + value + os.pathsep
  69     # Don't add the same value twice
  70     if val in env_val or env_val.startswith(value + os.pathsep):
  71         return
  72     env[var] = val + env_val
  73     env[var] = env[var].replace(os.pathsep + os.pathsep, os.pathsep).strip(os.pathsep)
  74
  75 def is_library_target_and_not_plugin(target, filename):
  76     '''
  77     Don't add plugins to PATH/LD_LIBRARY_PATH because:
  78     1. We don't need to
  79     2. It causes us to exceed the PATH length limit on Windows and Wine
  80     '''
  81     if not target['type'].startswith('shared'):
  82         return False
  83     if not target['installed']:
  84         return False
  85     # Check if this output of that target is a shared library
  86     if not SHAREDLIB_REG.search(filename):
  87         return False
  88     # Check if it's installed to the gstreamer plugin location
  89     for install_filename in listify(target['install_filename']):
  90         if install_filename.endswith(os.path.basename(filename)):
  91             break
  92     else:
  93         # None of the installed files in the target correspond to the built
  94         # filename, so skip
  95         return False
  96
  97     global GSTPLUGIN_FILEPATH_REG
  98     if GSTPLUGIN_FILEPATH_REG is None:
  99         GSTPLUGIN_FILEPATH_REG = re.compile(GSTPLUGIN_FILEPATH_REG_TEMPLATE)
 100     if GSTPLUGIN_FILEPATH_REG.search(install_filename.replace('\\', '/')):
 101         return False
 102     return True
 103
 104 def is_binary_target_and_in_path(target, filename, bindir):
 105     if target['type'] != 'executable':
 106         return False
 107     if not target['installed']:
 108         return False
 109     # Check if this file installed by this target is installed to bindir
 110     for install_filename in listify(target['install_filename']):
 111         if install_filename.endswith(os.path.basename(filename)):
 112             break
 113     else:
 114         # None of the installed files in the target correspond to the built
 115         # filename, so skip
 116         return False
 117     fpath = PurePath(install_filename)
 118     if fpath.parent != bindir:
 119         return False
 120     return True
 121
 122
 123 def get_wine_subprocess_env(options, env):
 124     with open(os.path.join(options.builddir, 'meson-info', 'intro-buildoptions.json')) as f:
 125         buildoptions = json.load(f)
 126
 127     prefix, = [o for o in buildoptions if o['name'] == 'prefix']
 128     path = os.path.normpath(os.path.join(prefix['value'], 'bin'))
 129     prepend_env_var(env, "PATH", path, options.sysroot)
 130     wine_path = get_wine_shortpath(
 131         options.wine.split(' '),
 132         [path] + env.get('WINEPATH', '').split(';')
 133     )
 134     if options.winepath:
 135         wine_path += ';' + options.winepath
 136     env['WINEPATH'] = wine_path
 137     env['WINEDEBUG'] = 'fixme-all'
 138
 139     return env
 140
 141 def setup_gdb(options):
 142     python_paths = set()
 143
 144     if not shutil.which('gdb'):
 145         return python_paths
 146
 147     bdir = pathlib.Path(options.builddir).resolve()
 148     for libpath, gdb_path in [
 149             (os.path.join("subprojects", "gstreamer", "gst"),
 150              os.path.join("subprojects", "gstreamer", "libs", "gst", "helpers")),
 151             (os.path.join("subprojects", "glib", "gobject"), None),
 152             (os.path.join("subprojects", "glib", "glib"), None)]:
 153
 154         if not gdb_path:
 155             gdb_path = libpath
 156
 157         autoload_path = (pathlib.Path(bdir) / 'gdb-auto-load').joinpath(*bdir.parts[1:]) / libpath
 158         autoload_path.mkdir(parents=True, exist_ok=True)
 159         for gdb_helper in glob.glob(str(bdir / gdb_path / "*-gdb.py")):
 160             python_paths.add(str(bdir / gdb_path))
 161             python_paths.add(os.path.join(options.srcdir, gdb_path))
 162             try:
 163                 os.symlink(gdb_helper, str(autoload_path / os.path.basename(gdb_helper)))
 164             except FileExistsError:
 165                 pass
 166
 167     gdbinit_line = 'add-auto-load-scripts-directory {}\n'.format(bdir / 'gdb-auto-load')
 168     try:
 169         with open(os.path.join(options.srcdir, '.gdbinit'), 'r') as f:
 170             if gdbinit_line in f.readlines():
 171                 return python_paths
 172     except FileNotFoundError:
 173         pass
 174
 175     with open(os.path.join(options.srcdir, '.gdbinit'), 'a') as f:
 176         f.write(gdbinit_line)
 177
 178     return python_paths
 179
 180
 181 def get_subprocess_env(options, gst_version):
 182     env = os.environ.copy()
 183
 184     env["CURRENT_GST"] = os.path.normpath(SCRIPTDIR)
 185     env["GST_VERSION"] = gst_version
 186     env["GST_VALIDATE_SCENARIOS_PATH"] = os.path.normpath(
 187         "%s/subprojects/gst-devtools/validate/data/scenarios" % SCRIPTDIR)
 188     env["GST_VALIDATE_PLUGIN_PATH"] = os.path.normpath(
 189         "%s/subprojects/gst-devtools/validate/plugins" % options.builddir)
 190     env["GST_VALIDATE_APPS_DIR"] = os.path.normpath(
 191         "%s/subprojects/gst-editing-services/tests/validate" % SCRIPTDIR)
 192     env["GST_ENV"] = 'gst-' + gst_version
 193     env["GST_REGISTRY"] = os.path.normpath(options.builddir + "/registry.dat")
 194     prepend_env_var(env, "PATH", os.path.normpath(
 195         "%s/subprojects/gst-devtools/validate/tools" % options.builddir),
 196         options.sysroot)
 197
 198     if options.wine:
 199         return get_wine_subprocess_env(options, env)
 200
 201     prepend_env_var(env, "PATH", os.path.join(SCRIPTDIR, 'meson'),
 202         options.sysroot)
 203
 204     env["GST_PLUGIN_SYSTEM_PATH"] = ""
 205     env["GST_PLUGIN_SCANNER"] = os.path.normpath(
 206         "%s/subprojects/gstreamer/libs/gst/helpers/gst-plugin-scanner" % options.builddir)
 207     env["GST_PTP_HELPER"] = os.path.normpath(
 208         "%s/subprojects/gstreamer/libs/gst/helpers/gst-ptp-helper" % options.builddir)
 209
 210     if os.name == 'nt':
 211         lib_path_envvar = 'PATH'
 212     elif platform.system() == 'Darwin':
 213         lib_path_envvar = 'DYLD_LIBRARY_PATH'
 214     else:
 215         lib_path_envvar = 'LD_LIBRARY_PATH'
 216
 217     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(SCRIPTDIR, 'subprojects',
 218                                                         'gst-python', 'plugin'),
 219                     options.sysroot)
 220     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(PREFIX_DIR, 'lib',
 221                                                         'gstreamer-1.0'),
 222                     options.sysroot)
 223     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(options.builddir, 'subprojects',
 224                                                          'libnice', 'gst'),
 225                     options.sysroot)
 226     prepend_env_var(env, "GST_VALIDATE_SCENARIOS_PATH",
 227                     os.path.join(PREFIX_DIR, 'share', 'gstreamer-1.0',
 228                                  'validate', 'scenarios'),
 229                     options.sysroot)
 230     prepend_env_var(env, "GI_TYPELIB_PATH", os.path.join(PREFIX_DIR, 'lib',
 231                                                          'lib', 'girepository-1.0'),
 232                     options.sysroot)
 233     prepend_env_var(env, "PKG_CONFIG_PATH", os.path.join(PREFIX_DIR, 'lib', 'pkgconfig'),
 234                     options.sysroot)
 235
 236     # gst-indent
 237     prepend_env_var(env, "PATH", os.path.join(SCRIPTDIR, 'gstreamer', 'tools'),
 238                     options.sysroot)
 239
 240     # tools: gst-launch-1.0, gst-inspect-1.0
 241     prepend_env_var(env, "PATH", os.path.join(options.builddir, 'subprojects',
 242                                               'gstreamer', 'tools'),
 243                     options.sysroot)
 244     prepend_env_var(env, "PATH", os.path.join(options.builddir, 'subprojects',
 245                                               'gst-plugins-base', 'tools'),
 246                     options.sysroot)
 247
 248     # Library and binary search paths
 249     prepend_env_var(env, "PATH", os.path.join(PREFIX_DIR, 'bin'),
 250                     options.sysroot)
 251     if lib_path_envvar != 'PATH':
 252         prepend_env_var(env, lib_path_envvar, os.path.join(PREFIX_DIR, 'lib'),
 253                         options.sysroot)
 254         prepend_env_var(env, lib_path_envvar, os.path.join(PREFIX_DIR, 'lib64'),
 255                         options.sysroot)
 256     elif 'QMAKE' in os.environ:
 257         # There's no RPATH on Windows, so we need to set PATH for the qt5 DLLs
 258         prepend_env_var(env, 'PATH', os.path.dirname(os.environ['QMAKE']),
 259                         options.sysroot)
 260
 261     meson = get_meson()
 262     targets_s = subprocess.check_output(meson + ['introspect', options.builddir, '--targets'])
 263     targets = json.loads(targets_s.decode())
 264     paths = set()
 265     mono_paths = set()
 266     srcdir_path = pathlib.Path(options.srcdir)
 267
 268     build_options_s = subprocess.check_output(meson + ['introspect', options.builddir, '--buildoptions'])
 269     build_options = json.loads(build_options_s.decode())
 270     libdir, = [o['value'] for o in build_options if o['name'] == 'libdir']
 271     libdir = PurePath(libdir)
 272     prefix, = [o['value'] for o in build_options if o['name'] == 'prefix']
 273     bindir, = [o['value'] for o in build_options if o['name'] == 'bindir']
 274     prefix = PurePath(prefix)
 275     bindir = prefix / bindir
 276
 277     global GSTPLUGIN_FILEPATH_REG_TEMPLATE
 278     GSTPLUGIN_FILEPATH_REG_TEMPLATE = GSTPLUGIN_FILEPATH_REG_TEMPLATE.format(libdir=libdir.as_posix())
 279
 280     for target in targets:
 281         filenames = listify(target['filename'])
 282         for filename in filenames:
 283             root = os.path.dirname(filename)
 284             if srcdir_path / "subprojects/gst-devtools/validate/plugins" in (srcdir_path / root).parents:
 285                 continue
 286             if filename.endswith('.dll'):
 287                 mono_paths.add(os.path.join(options.builddir, root))
 288             if TYPELIB_REG.search(filename):
 289                 prepend_env_var(env, "GI_TYPELIB_PATH",
 290                                 os.path.join(options.builddir, root),
 291                                 options.sysroot)
 292             elif is_library_target_and_not_plugin(target, filename):
 293                 prepend_env_var(env, lib_path_envvar,
 294                                 os.path.join(options.builddir, root),
 295                                 options.sysroot)
 296             elif is_binary_target_and_in_path(target, filename, bindir):
 297                 paths.add(os.path.join(options.builddir, root))
 298
 299     with open(os.path.join(options.builddir, 'GstPluginsPath.json')) as f:
 300         for plugin_path in json.load(f):
 301             prepend_env_var(env, 'GST_PLUGIN_PATH', plugin_path,
 302                             options.sysroot)
 303
 304     for p in paths:
 305         prepend_env_var(env, 'PATH', p, options.sysroot)
 306
 307     if os.name != 'nt':
 308         for p in mono_paths:
 309             prepend_env_var(env, "MONO_PATH", p, options.sysroot)
 310
 311     presets = set()
 312     encoding_targets = set()
 313     pkg_dirs = set()
 314     python_dirs = setup_gdb(options)
 315     if '--installed' in subprocess.check_output(meson + ['introspect', '-h']).decode():
 316         installed_s = subprocess.check_output(meson + ['introspect', options.builddir, '--installed'])
 317         for path, installpath in json.loads(installed_s.decode()).items():
 318             installpath_parts = pathlib.Path(installpath).parts
 319             path_parts = pathlib.Path(path).parts
 320
 321             # We want to add all python modules to the PYTHONPATH
 322             # in a manner consistent with the way they would be imported:
 323             # For example if the source path /home/meh/foo/bar.py
 324             # is to be installed in /usr/lib/python/site-packages/foo/bar.py,
 325             # we want to add /home/meh to the PYTHONPATH.
 326             # This will only work for projects where the paths to be installed
 327             # mirror the installed directory layout, for example if the path
 328             # is /home/meh/baz/bar.py and the install path is
 329             # /usr/lib/site-packages/foo/bar.py , we will not add anything
 330             # to PYTHONPATH, but the current approach works with pygobject
 331             # and gst-python at least.
 332             if 'site-packages' in installpath_parts:
 333                 install_subpath = os.path.join(*installpath_parts[installpath_parts.index('site-packages') + 1:])
 334                 if path.endswith(install_subpath):
 335                     python_dirs.add(path[:len (install_subpath) * -1])
 336
 337             if path.endswith('.prs'):
 338                 presets.add(os.path.dirname(path))
 339             elif path.endswith('.gep'):
 340                 encoding_targets.add(
 341                     os.path.abspath(os.path.join(os.path.dirname(path), '..')))
 342             elif path.endswith('.pc'):
 343                 # Is there a -uninstalled pc file for this file?
 344                 uninstalled = "{0}-uninstalled.pc".format(path[:-3])
 345                 if os.path.exists(uninstalled):
 346                     pkg_dirs.add(os.path.dirname(path))
 347
 348             if path.endswith('gstomx.conf'):
 349                 prepend_env_var(env, 'GST_OMX_CONFIG_DIR', os.path.dirname(path),
 350                                 options.sysroot)
 351
 352         for p in presets:
 353             prepend_env_var(env, 'GST_PRESET_PATH', p, options.sysroot)
 354
 355         for t in encoding_targets:
 356             prepend_env_var(env, 'GST_ENCODING_TARGET_PATH', t, options.sysroot)
 357
 358         for pkg_dir in pkg_dirs:
 359             prepend_env_var(env, "PKG_CONFIG_PATH", pkg_dir, options.sysroot)
 360
 361     # Check if meson has generated -uninstalled pkgconfig files
 362     meson_uninstalled = pathlib.Path(options.builddir) / 'meson-uninstalled'
 363     if meson_uninstalled.is_dir():
 364         prepend_env_var(env, 'PKG_CONFIG_PATH', str(meson_uninstalled), options.sysroot)
 365
 366     for python_dir in python_dirs:
 367         prepend_env_var(env, 'PYTHONPATH', python_dir, options.sysroot)
 368
 369     mesonpath = os.path.join(SCRIPTDIR, "meson")
 370     if os.path.join(mesonpath):
 371         # Add meson/ into PYTHONPATH if we are using a local meson
 372         prepend_env_var(env, 'PYTHONPATH', mesonpath, options.sysroot)
 373
 374     # For devhelp books
 375     if 'XDG_DATA_DIRS' not in env or not env['XDG_DATA_DIRS']:
 376         # Preserve default paths when empty
 377         prepend_env_var(env, 'XDG_DATA_DIRS', '/usr/local/share/:/usr/share/', '')
 378
 379     prepend_env_var (env, 'XDG_DATA_DIRS', os.path.join(options.builddir,
 380                                                         'subprojects',
 381                                                         'gst-docs',
 382                                                         'GStreamer-doc'),
 383                      options.sysroot)
 384
 385     if 'XDG_CONFIG_DIRS' not in env or not env['XDG_CONFIG_DIRS']:
 386         # Preserve default paths when empty
 387         prepend_env_var(env, 'XDG_CONFIG_DIRS', '/etc/local/xdg:/etc/xdg', '')
 388
 389     prepend_env_var(env, "XDG_CONFIG_DIRS", os.path.join(PREFIX_DIR, 'etc', 'xdg'),
 390                     options.sysroot)
 391
 392     return env
 393
 394 def get_windows_shell():
 395     command = ['powershell.exe' ,'-noprofile', '-executionpolicy', 'bypass', '-file', 'cmd_or_ps.ps1']
 396     result = subprocess.check_output(command)
 397     return result.decode().strip()
 398
 399 if __name__ == "__main__":
 400     parser = argparse.ArgumentParser(prog="gst-env")
 401
 402     parser.add_argument("--builddir",
 403                         default=DEFAULT_BUILDDIR,
 404                         help="The meson build directory")
 405     parser.add_argument("--srcdir",
 406                         default=SCRIPTDIR,
 407                         help="The top level source directory")
 408     parser.add_argument("--sysroot",
 409                         default='',
 410                         help="The sysroot path used during cross-compilation")
 411     parser.add_argument("--wine",
 412                         default='',
 413                         help="Build a wine env based on specified wine command")
 414     parser.add_argument("--winepath",
 415                         default='',
 416                         help="Extra path to set to WINEPATH.")
 417     parser.add_argument("--only-environment",
 418                         action='store_true',
 419                         default=False,
 420                         help="Do not start a shell, only print required environment.")
 421     options, args = parser.parse_known_args()
 422
 423     if not os.path.exists(options.builddir):
 424         print("GStreamer not built in %s\n\nBuild it and try again" %
 425               options.builddir)
 426         exit(1)
 427     options.builddir = os.path.abspath(options.builddir)
 428
 429     if not os.path.exists(options.srcdir):
 430         print("The specified source dir does not exist" %
 431               options.srcdir)
 432         exit(1)
 433
 434     # The following incantation will retrieve the current branch name.
 435     try:
 436       gst_version = git("rev-parse", "--symbolic-full-name", "--abbrev-ref", "HEAD",
 437                         repository_path=options.srcdir).strip('\n')
 438     except subprocess.CalledProcessError:
 439       gst_version = "unknown"
 440
 441     if options.wine:
 442         gst_version += '-' + os.path.basename(options.wine)
 443
 444     env = get_subprocess_env(options, gst_version)
 445     if not args:
 446         if os.name == 'nt':
 447             shell = get_windows_shell()
 448             if shell == 'powershell.exe':
 449                 args = ['powershell.exe']
 450                 args += ['-NoLogo', '-NoExit']
 451                 prompt = 'function global:prompt {  "[gst-' + gst_version + '"+"] PS " + $PWD + "> "}'
 452                 args += ['-Command', prompt]
 453             else:
 454                 args = [os.environ.get("COMSPEC", r"C:\WINDOWS\system32\cmd.exe")]
 455                 args += ['/k', 'prompt [gst-{}] $P$G'.format(gst_version)]
 456         else:
 457             args = [os.environ.get("SHELL", os.path.realpath("/bin/sh"))]
 458         if args[0].endswith('bash') and not strtobool(os.environ.get("GST_BUILD_DISABLE_PS1_OVERRIDE", r"FALSE")):
 459             # Let the GC remove the tmp file
 460             tmprc = tempfile.NamedTemporaryFile(mode='w')
 461             bashrc = os.path.expanduser('~/.bashrc')
 462             if os.path.exists(bashrc):
 463                 with open(bashrc, 'r') as src:
 464                     shutil.copyfileobj(src, tmprc)
 465             tmprc.write('\nexport PS1="[gst-%s] $PS1"' % gst_version)
 466             tmprc.flush()
 467             args.append("--rcfile")
 468             args.append(tmprc.name)
 469         elif args[0].endswith('fish'):
 470             # Ignore SIGINT while using fish as the shell to make it behave
 471             # like other shells such as bash and zsh.
 472             # See: https://gitlab.freedesktop.org/gstreamer/gst-build/issues/18
 473             signal.signal(signal.SIGINT, lambda x, y: True)
 474             # Set the prompt
 475             args.append('--init-command')
 476             prompt_cmd = '''functions --copy fish_prompt original_fish_prompt
 477             function fish_prompt
 478                 echo -n '[gst-{}] '(original_fish_prompt)
 479             end'''.format(gst_version)
 480             args.append(prompt_cmd)
 481         elif args[0].endswith('zsh'):
 482             tmpdir = tempfile.TemporaryDirectory()
 483             # Let the GC remove the tmp file
 484             tmprc = open(os.path.join(tmpdir.name, '.zshrc'), 'w')
 485             zshrc = os.path.expanduser('~/.zshrc')
 486             if os.path.exists(zshrc):
 487                 with open(zshrc, 'r') as src:
 488                     shutil.copyfileobj(src, tmprc)
 489             tmprc.write('\nexport PROMPT="[gst-{}] $PROMPT"'.format(gst_version))
 490             tmprc.flush()
 491             env['ZDOTDIR'] = tmpdir.name
 492     try:
 493         if options.only_environment:
 494             for name, value in env.items():
 495                 print('{}={}'.format(name, shlex.quote(value)))
 496                 print('export {}'.format(name))
 497         else:
 498             exit(subprocess.call(args, close_fds=False, env=env))
 499
 500     except subprocess.CalledProcessError as e:
 501         exit(e.returncode)