gst-env.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import contextlib
   5 import glob
   6 import json
   7 import os
   8 import platform
   9 import re
  10 import site
  11 import shlex
  12 import shutil
  13 import subprocess
  14 import sys
  15 import tempfile
  16 import pathlib
  17 import signal
  18 from pathlib import PurePath
  19
  20 from distutils.sysconfig import get_python_lib
  21 from distutils.util import strtobool
  22
  23 from scripts.common import get_meson
  24 from scripts.common import git
  25 from scripts.common import win32_get_short_path_name
  26 from scripts.common import get_wine_shortpath
  27
  28 SCRIPTDIR = os.path.dirname(os.path.realpath(__file__))
  29 PREFIX_DIR = os.path.join(SCRIPTDIR, 'prefix')
  30 # Use '_build' as the builddir instead of 'build'
  31 DEFAULT_BUILDDIR = os.path.join(SCRIPTDIR, 'build')
  32 if not os.path.exists(DEFAULT_BUILDDIR):
  33     DEFAULT_BUILDDIR = os.path.join(SCRIPTDIR, '_build')
  34
  35 TYPELIB_REG = re.compile(r'.*\.typelib$')
  36 SHAREDLIB_REG = re.compile(r'\.so|\.dylib|\.dll')
  37
  38 # libdir is expanded from option of the same name listed in the `meson
  39 # introspect --buildoptions` output.
  40 GSTPLUGIN_FILEPATH_REG_TEMPLATE = r'.*/{libdir}/gstreamer-1.0/[^/]+$'
  41 GSTPLUGIN_FILEPATH_REG = None
  42
  43 def listify(o):
  44     if isinstance(o, str):
  45         return [o]
  46     if isinstance(o, list):
  47         return o
  48     raise AssertionError('Object {!r} must be a string or a list'.format(o))
  49
  50 def stringify(o):
  51     if isinstance(o, str):
  52         return o
  53     if isinstance(o, list):
  54         if len(o) == 1:
  55             return o[0]
  56         raise AssertionError('Did not expect object {!r} to have more than one element'.format(o))
  57     raise AssertionError('Object {!r} must be a string or a list'.format(o))
  58
  59 def prepend_env_var(env, var, value, sysroot):
  60     if value.startswith(sysroot):
  61         value = value[len(sysroot):]
  62     # Try not to exceed maximum length limits for env vars on Windows
  63     if os.name == 'nt':
  64         value = win32_get_short_path_name(value)
  65     env_val = env.get(var, '')
  66     val = os.pathsep + value + os.pathsep
  67     # Don't add the same value twice
  68     if val in env_val or env_val.startswith(value + os.pathsep):
  69         return
  70     env[var] = val + env_val
  71     env[var] = env[var].replace(os.pathsep + os.pathsep, os.pathsep).strip(os.pathsep)
  72
  73 def is_library_target_and_not_plugin(target, filename):
  74     '''
  75     Don't add plugins to PATH/LD_LIBRARY_PATH because:
  76     1. We don't need to
  77     2. It causes us to exceed the PATH length limit on Windows and Wine
  78     '''
  79     if not target['type'].startswith('shared'):
  80         return False
  81     if not target['installed']:
  82         return False
  83     # Check if this output of that target is a shared library
  84     if not SHAREDLIB_REG.search(filename):
  85         return False
  86     # Check if it's installed to the gstreamer plugin location
  87     for install_filename in listify(target['install_filename']):
  88         if install_filename.endswith(os.path.basename(filename)):
  89             break
  90     else:
  91         # None of the installed files in the target correspond to the built
  92         # filename, so skip
  93         return False
  94
  95     global GSTPLUGIN_FILEPATH_REG
  96     if GSTPLUGIN_FILEPATH_REG is None:
  97         GSTPLUGIN_FILEPATH_REG = re.compile(GSTPLUGIN_FILEPATH_REG_TEMPLATE)
  98     if GSTPLUGIN_FILEPATH_REG.search(install_filename.replace('\\', '/')):
  99         return False
 100     return True
 101
 102 def is_binary_target_and_in_path(target, filename, bindir):
 103     if target['type'] != 'executable':
 104         return False
 105     if not target['installed']:
 106         return False
 107     # Check if this file installed by this target is installed to bindir
 108     for install_filename in listify(target['install_filename']):
 109         if install_filename.endswith(os.path.basename(filename)):
 110             break
 111     else:
 112         # None of the installed files in the target correspond to the built
 113         # filename, so skip
 114         return False
 115     fpath = PurePath(install_filename)
 116     if fpath.parent != bindir:
 117         return False
 118     return True
 119
 120
 121 def get_wine_subprocess_env(options, env):
 122     with open(os.path.join(options.builddir, 'meson-info', 'intro-buildoptions.json')) as f:
 123         buildoptions = json.load(f)
 124
 125     prefix, = [o for o in buildoptions if o['name'] == 'prefix']
 126     path = os.path.normpath(os.path.join(prefix['value'], 'bin'))
 127     prepend_env_var(env, "PATH", path, options.sysroot)
 128     wine_path = get_wine_shortpath(
 129         options.wine.split(' '),
 130         [path] + env.get('WINEPATH', '').split(';')
 131     )
 132     if options.winepath:
 133         wine_path += ';' + options.winepath
 134     env['WINEPATH'] = wine_path
 135     env['WINEDEBUG'] = 'fixme-all'
 136
 137     return env
 138
 139 def setup_gdb(options):
 140     bdir = os.path.realpath(options.builddir)
 141     python_paths = set()
 142     for libpath, gdb_path in [
 143             ("subprojects/gstreamer/gst/", "subprojects/gstreamer/libs/gst/helpers/"),
 144             ("subprojects/glib/gobject", None),
 145             ("subprojects/glib/glib", None)]:
 146
 147         if not gdb_path:
 148             gdb_path = libpath
 149
 150         autoload_path = os.path.join(bdir, "gdb-auto-load/", bdir[1:], libpath)
 151         os.makedirs(autoload_path, exist_ok=True)
 152         for gdb_helper in glob.glob(os.path.join(bdir, gdb_path, "*-gdb.py")):
 153             python_paths.add(os.path.join(bdir, gdb_path))
 154             python_paths.add(os.path.join(options.srcdir, gdb_path))
 155             try:
 156                 os.symlink(gdb_helper, os.path.join(autoload_path, os.path.basename(gdb_helper)))
 157             except FileExistsError:
 158                 pass
 159
 160     gdbinit_line = 'add-auto-load-scripts-directory %s' % os.path.join(bdir, 'gdb-auto-load\n')
 161     try:
 162         with open(os.path.join(options.srcdir, '.gdbinit'), 'r') as f:
 163             if gdbinit_line in f.readlines():
 164                 return python_paths
 165     except FileNotFoundError:
 166         pass
 167
 168     with open(os.path.join(options.srcdir, '.gdbinit'), 'a') as f:
 169         f.write(gdbinit_line)
 170
 171     return python_paths
 172
 173
 174 def get_subprocess_env(options, gst_version):
 175     env = os.environ.copy()
 176
 177     env["CURRENT_GST"] = os.path.normpath(SCRIPTDIR)
 178     env["GST_VERSION"] = gst_version
 179     env["GST_VALIDATE_SCENARIOS_PATH"] = os.path.normpath(
 180         "%s/subprojects/gst-devtools/validate/data/scenarios" % SCRIPTDIR)
 181     env["GST_VALIDATE_PLUGIN_PATH"] = os.path.normpath(
 182         "%s/subprojects/gst-devtools/validate/plugins" % options.builddir)
 183     env["GST_VALIDATE_APPS_DIR"] = os.path.normpath(
 184         "%s/subprojects/gst-editing-services/tests/validate" % SCRIPTDIR)
 185     env["GST_ENV"] = 'gst-' + gst_version
 186     env["GST_REGISTRY"] = os.path.normpath(options.builddir + "/registry.dat")
 187     prepend_env_var(env, "PATH", os.path.normpath(
 188         "%s/subprojects/gst-devtools/validate/tools" % options.builddir),
 189         options.sysroot)
 190
 191     if options.wine:
 192         return get_wine_subprocess_env(options, env)
 193
 194     prepend_env_var(env, "PATH", os.path.join(SCRIPTDIR, 'meson'),
 195         options.sysroot)
 196
 197     env["GST_PLUGIN_SYSTEM_PATH"] = ""
 198     env["GST_PLUGIN_SCANNER"] = os.path.normpath(
 199         "%s/subprojects/gstreamer/libs/gst/helpers/gst-plugin-scanner" % options.builddir)
 200     env["GST_PTP_HELPER"] = os.path.normpath(
 201         "%s/subprojects/gstreamer/libs/gst/helpers/gst-ptp-helper" % options.builddir)
 202
 203     if os.name == 'nt':
 204         lib_path_envvar = 'PATH'
 205     elif platform.system() == 'Darwin':
 206         lib_path_envvar = 'DYLD_LIBRARY_PATH'
 207     else:
 208         lib_path_envvar = 'LD_LIBRARY_PATH'
 209
 210     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(SCRIPTDIR, 'subprojects',
 211                                                         'gst-python', 'plugin'),
 212                     options.sysroot)
 213     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(PREFIX_DIR, 'lib',
 214                                                         'gstreamer-1.0'),
 215                     options.sysroot)
 216     prepend_env_var(env, "GST_PLUGIN_PATH", os.path.join(options.builddir, 'subprojects',
 217                                                          'libnice', 'gst'),
 218                     options.sysroot)
 219     prepend_env_var(env, "GST_VALIDATE_SCENARIOS_PATH",
 220                     os.path.join(PREFIX_DIR, 'share', 'gstreamer-1.0',
 221                                  'validate', 'scenarios'),
 222                     options.sysroot)
 223     prepend_env_var(env, "GI_TYPELIB_PATH", os.path.join(PREFIX_DIR, 'lib',
 224                                                          'lib', 'girepository-1.0'),
 225                     options.sysroot)
 226     prepend_env_var(env, "PKG_CONFIG_PATH", os.path.join(PREFIX_DIR, 'lib', 'pkgconfig'),
 227                     options.sysroot)
 228
 229     # gst-indent
 230     prepend_env_var(env, "PATH", os.path.join(SCRIPTDIR, 'gstreamer', 'tools'),
 231                     options.sysroot)
 232
 233     # tools: gst-launch-1.0, gst-inspect-1.0
 234     prepend_env_var(env, "PATH", os.path.join(options.builddir, 'subprojects',
 235                                               'gstreamer', 'tools'),
 236                     options.sysroot)
 237     prepend_env_var(env, "PATH", os.path.join(options.builddir, 'subprojects',
 238                                               'gst-plugins-base', 'tools'),
 239                     options.sysroot)
 240
 241     # Library and binary search paths
 242     prepend_env_var(env, "PATH", os.path.join(PREFIX_DIR, 'bin'),
 243                     options.sysroot)
 244     if lib_path_envvar != 'PATH':
 245         prepend_env_var(env, lib_path_envvar, os.path.join(PREFIX_DIR, 'lib'),
 246                         options.sysroot)
 247         prepend_env_var(env, lib_path_envvar, os.path.join(PREFIX_DIR, 'lib64'),
 248                         options.sysroot)
 249     elif 'QMAKE' in os.environ:
 250         # There's no RPATH on Windows, so we need to set PATH for the qt5 DLLs
 251         prepend_env_var(env, 'PATH', os.path.dirname(os.environ['QMAKE']),
 252                         options.sysroot)
 253
 254     meson = get_meson()
 255     targets_s = subprocess.check_output(meson + ['introspect', options.builddir, '--targets'])
 256     targets = json.loads(targets_s.decode())
 257     paths = set()
 258     mono_paths = set()
 259     srcdir_path = pathlib.Path(options.srcdir)
 260
 261     build_options_s = subprocess.check_output(meson + ['introspect', options.builddir, '--buildoptions'])
 262     build_options = json.loads(build_options_s.decode())
 263     libdir, = [o['value'] for o in build_options if o['name'] == 'libdir']
 264     libdir = PurePath(libdir)
 265     prefix, = [o['value'] for o in build_options if o['name'] == 'prefix']
 266     bindir, = [o['value'] for o in build_options if o['name'] == 'bindir']
 267     prefix = PurePath(prefix)
 268     bindir = prefix / bindir
 269
 270     global GSTPLUGIN_FILEPATH_REG_TEMPLATE
 271     GSTPLUGIN_FILEPATH_REG_TEMPLATE = GSTPLUGIN_FILEPATH_REG_TEMPLATE.format(libdir=libdir.as_posix())
 272
 273     for target in targets:
 274         filenames = listify(target['filename'])
 275         for filename in filenames:
 276             root = os.path.dirname(filename)
 277             if srcdir_path / "subprojects/gst-devtools/validate/plugins" in (srcdir_path / root).parents:
 278                 continue
 279             if filename.endswith('.dll'):
 280                 mono_paths.add(os.path.join(options.builddir, root))
 281             if TYPELIB_REG.search(filename):
 282                 prepend_env_var(env, "GI_TYPELIB_PATH",
 283                                 os.path.join(options.builddir, root),
 284                                 options.sysroot)
 285             elif is_library_target_and_not_plugin(target, filename):
 286                 prepend_env_var(env, lib_path_envvar,
 287                                 os.path.join(options.builddir, root),
 288                                 options.sysroot)
 289             elif is_binary_target_and_in_path(target, filename, bindir):
 290                 paths.add(os.path.join(options.builddir, root))
 291
 292     with open(os.path.join(options.builddir, 'GstPluginsPath.json')) as f:
 293         for plugin_path in json.load(f):
 294             prepend_env_var(env, 'GST_PLUGIN_PATH', plugin_path,
 295                             options.sysroot)
 296
 297     for p in paths:
 298         prepend_env_var(env, 'PATH', p, options.sysroot)
 299
 300     if os.name != 'nt':
 301         for p in mono_paths:
 302             prepend_env_var(env, "MONO_PATH", p, options.sysroot)
 303
 304     presets = set()
 305     encoding_targets = set()
 306     pkg_dirs = set()
 307     python_dirs = setup_gdb(options)
 308     if '--installed' in subprocess.check_output(meson + ['introspect', '-h']).decode():
 309         installed_s = subprocess.check_output(meson + ['introspect', options.builddir, '--installed'])
 310         for path, installpath in json.loads(installed_s.decode()).items():
 311             installpath_parts = pathlib.Path(installpath).parts
 312             path_parts = pathlib.Path(path).parts
 313
 314             # We want to add all python modules to the PYTHONPATH
 315             # in a manner consistent with the way they would be imported:
 316             # For example if the source path /home/meh/foo/bar.py
 317             # is to be installed in /usr/lib/python/site-packages/foo/bar.py,
 318             # we want to add /home/meh to the PYTHONPATH.
 319             # This will only work for projects where the paths to be installed
 320             # mirror the installed directory layout, for example if the path
 321             # is /home/meh/baz/bar.py and the install path is
 322             # /usr/lib/site-packages/foo/bar.py , we will not add anything
 323             # to PYTHONPATH, but the current approach works with pygobject
 324             # and gst-python at least.
 325             if 'site-packages' in installpath_parts:
 326                 install_subpath = os.path.join(*installpath_parts[installpath_parts.index('site-packages') + 1:])
 327                 if path.endswith(install_subpath):
 328                     python_dirs.add(path[:len (install_subpath) * -1])
 329
 330             if path.endswith('.prs'):
 331                 presets.add(os.path.dirname(path))
 332             elif path.endswith('.gep'):
 333                 encoding_targets.add(
 334                     os.path.abspath(os.path.join(os.path.dirname(path), '..')))
 335             elif path.endswith('.pc'):
 336                 # Is there a -uninstalled pc file for this file?
 337                 uninstalled = "{0}-uninstalled.pc".format(path[:-3])
 338                 if os.path.exists(uninstalled):
 339                     pkg_dirs.add(os.path.dirname(path))
 340
 341             if path.endswith('gstomx.conf'):
 342                 prepend_env_var(env, 'GST_OMX_CONFIG_DIR', os.path.dirname(path),
 343                                 options.sysroot)
 344
 345         for p in presets:
 346             prepend_env_var(env, 'GST_PRESET_PATH', p, options.sysroot)
 347
 348         for t in encoding_targets:
 349             prepend_env_var(env, 'GST_ENCODING_TARGET_PATH', t, options.sysroot)
 350
 351         for pkg_dir in pkg_dirs:
 352             prepend_env_var(env, "PKG_CONFIG_PATH", pkg_dir, options.sysroot)
 353     prepend_env_var(env, "PKG_CONFIG_PATH", os.path.join(options.builddir,
 354                                                          'subprojects',
 355                                                          'gst-plugins-good',
 356                                                          'pkgconfig'),
 357                     options.sysroot)
 358
 359     for python_dir in python_dirs:
 360         prepend_env_var(env, 'PYTHONPATH', python_dir, options.sysroot)
 361
 362     mesonpath = os.path.join(SCRIPTDIR, "meson")
 363     if os.path.join(mesonpath):
 364         # Add meson/ into PYTHONPATH if we are using a local meson
 365         prepend_env_var(env, 'PYTHONPATH', mesonpath, options.sysroot)
 366
 367     # For devhelp books
 368     if 'XDG_DATA_DIRS' not in env or not env['XDG_DATA_DIRS']:
 369         # Preserve default paths when empty
 370         prepend_env_var(env, 'XDG_DATA_DIRS', '/usr/local/share/:/usr/share/', '')
 371
 372     prepend_env_var (env, 'XDG_DATA_DIRS', os.path.join(options.builddir,
 373                                                         'subprojects',
 374                                                         'gst-docs',
 375                                                         'GStreamer-doc'),
 376                      options.sysroot)
 377
 378     if 'XDG_CONFIG_DIRS' not in env or not env['XDG_CONFIG_DIRS']:
 379         # Preserve default paths when empty
 380         prepend_env_var(env, 'XDG_CONFIG_DIRS', '/etc/local/xdg:/etc/xdg', '')
 381
 382     prepend_env_var(env, "XDG_CONFIG_DIRS", os.path.join(PREFIX_DIR, 'etc', 'xdg'),
 383                     options.sysroot)
 384
 385     return env
 386
 387 def get_windows_shell():
 388     command = ['powershell.exe' ,'-noprofile', '-executionpolicy', 'bypass', '-file', 'cmd_or_ps.ps1']
 389     result = subprocess.check_output(command)
 390     return result.decode().strip()
 391
 392 if __name__ == "__main__":
 393     parser = argparse.ArgumentParser(prog="gstreamer-uninstalled")
 394
 395     parser.add_argument("--builddir",
 396                         default=DEFAULT_BUILDDIR,
 397                         help="The meson build directory")
 398     parser.add_argument("--srcdir",
 399                         default=SCRIPTDIR,
 400                         help="The top level source directory")
 401     parser.add_argument("--sysroot",
 402                         default='',
 403                         help="The sysroot path used during cross-compilation")
 404     parser.add_argument("--wine",
 405                         default='',
 406                         help="Build a wine env based on specified wine command")
 407     parser.add_argument("--winepath",
 408                         default='',
 409                         help="Extra path to set to WINEPATH.")
 410     parser.add_argument("--only-environment",
 411                         action='store_true',
 412                         default=False,
 413                         help="Do not start a shell, only print required environment.")
 414     options, args = parser.parse_known_args()
 415
 416     if not os.path.exists(options.builddir):
 417         print("GStreamer not built in %s\n\nBuild it and try again" %
 418               options.builddir)
 419         exit(1)
 420     options.builddir = os.path.abspath(options.builddir)
 421
 422     if not os.path.exists(options.srcdir):
 423         print("The specified source dir does not exist" %
 424               options.srcdir)
 425         exit(1)
 426
 427     # The following incantation will retrieve the current branch name.
 428     gst_version = git("rev-parse", "--symbolic-full-name", "--abbrev-ref", "HEAD",
 429                       repository_path=options.srcdir).strip('\n')
 430
 431     if options.wine:
 432         gst_version += '-' + os.path.basename(options.wine)
 433
 434     env = get_subprocess_env(options, gst_version)
 435     if not args:
 436         if os.name == 'nt':
 437             shell = get_windows_shell()
 438             if shell == 'powershell.exe':
 439                 args = ['powershell.exe']
 440                 args += ['-NoLogo', '-NoExit']
 441                 prompt = 'function global:prompt {  "[gst-' + gst_version + '"+"] PS " + $PWD + "> "}'
 442                 args += ['-Command', prompt]
 443             else:
 444                 args = [os.environ.get("COMSPEC", r"C:\WINDOWS\system32\cmd.exe")]
 445                 args += ['/k', 'prompt [gst-{}] $P$G'.format(gst_version)]
 446         else:
 447             args = [os.environ.get("SHELL", os.path.realpath("/bin/sh"))]
 448         if args[0].endswith('bash') and not strtobool(os.environ.get("GST_BUILD_DISABLE_PS1_OVERRIDE", r"FALSE")):
 449             # Let the GC remove the tmp file
 450             tmprc = tempfile.NamedTemporaryFile(mode='w')
 451             bashrc = os.path.expanduser('~/.bashrc')
 452             if os.path.exists(bashrc):
 453                 with open(bashrc, 'r') as src:
 454                     shutil.copyfileobj(src, tmprc)
 455             tmprc.write('\nexport PS1="[gst-%s] $PS1"' % gst_version)
 456             tmprc.flush()
 457             args.append("--rcfile")
 458             args.append(tmprc.name)
 459         elif args[0].endswith('fish'):
 460             # Ignore SIGINT while using fish as the shell to make it behave
 461             # like other shells such as bash and zsh.
 462             # See: https://gitlab.freedesktop.org/gstreamer/gst-build/issues/18
 463             signal.signal(signal.SIGINT, lambda x, y: True)
 464             # Set the prompt
 465             args.append('--init-command')
 466             prompt_cmd = '''functions --copy fish_prompt original_fish_prompt
 467             function fish_prompt
 468                 echo -n '[gst-{}] '(original_fish_prompt)
 469             end'''.format(gst_version)
 470             args.append(prompt_cmd)
 471         elif args[0].endswith('zsh'):
 472             tmpdir = tempfile.TemporaryDirectory()
 473             # Let the GC remove the tmp file
 474             tmprc = open(os.path.join(tmpdir.name, '.zshrc'), 'w')
 475             zshrc = os.path.expanduser('~/.zshrc')
 476             if os.path.exists(zshrc):
 477                 with open(zshrc, 'r') as src:
 478                     shutil.copyfileobj(src, tmprc)
 479             tmprc.write('\nexport PROMPT="[gst-{}] $PROMPT"'.format(gst_version))
 480             tmprc.flush()
 481             env['ZDOTDIR'] = tmpdir.name
 482     try:
 483         if options.only_environment:
 484             for name, value in env.items():
 485                 print('{}={}'.format(name, shlex.quote(value)))
 486                 print('export {}'.format(name))
 487         else:
 488             exit(subprocess.call(args, close_fds=False, env=env))
 489
 490     except subprocess.CalledProcessError as e:
 491         exit(e.returncode)