Fix for x86_64 build fail
[platform/upstream/connectedhomeip.git] / third_party / pigweed / repo / pw_env_setup / py / pw_env_setup / environment.py
1 # Copyright 2020 The Pigweed Authors
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
4 # use this file except in compliance with the License. You may obtain a copy of
5 # the License at
6 #
7 #     https://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations under
13 # the License.
14 """Stores the environment changes necessary for Pigweed."""
15
16 import contextlib
17 import json
18 import os
19 import re
20
21 # The order here is important. On Python 2 we want StringIO.StringIO and not
22 # io.StringIO. On Python 3 there is no StringIO module so we want io.StringIO.
23 # Not using six because six is not a standard package we can expect to have
24 # installed in the system Python.
25 try:
26     from StringIO import StringIO  # type: ignore
27 except ImportError:
28     from io import StringIO
29
30 # Disable super() warnings since this file must be Python 2 compatible.
31 # pylint: disable=super-with-arguments
32
33 # goto label written to the end of Windows batch files for exiting a script.
34 _SCRIPT_END_LABEL = '_pw_end'
35
36
37 class BadNameType(TypeError):
38     pass
39
40
41 class BadValueType(TypeError):
42     pass
43
44
45 class EmptyValue(ValueError):
46     pass
47
48
49 class NewlineInValue(TypeError):
50     pass
51
52
53 class BadVariableName(ValueError):
54     pass
55
56
57 class UnexpectedAction(ValueError):
58     pass
59
60
61 class _Action(object):  # pylint: disable=useless-object-inheritance
62     def unapply(self, env, orig_env):
63         pass
64
65     def json(self, data):
66         pass
67
68     def write_deactivate(self,
69                          outs,
70                          windows=(os.name == 'nt'),
71                          replacements=()):
72         pass
73
74
75 class _VariableAction(_Action):
76     # pylint: disable=keyword-arg-before-vararg
77     def __init__(self, name, value, allow_empty_values=False, *args, **kwargs):
78         super(_VariableAction, self).__init__(*args, **kwargs)
79         self.name = name
80         self.value = value
81         self.allow_empty_values = allow_empty_values
82
83         self._check()
84
85     def _check(self):
86         try:
87             # In python2, unicode is a distinct type.
88             valid_types = (str, unicode)
89         except NameError:
90             valid_types = (str, )
91
92         if not isinstance(self.name, valid_types):
93             raise BadNameType('variable name {!r} not of type str'.format(
94                 self.name))
95         if not isinstance(self.value, valid_types):
96             raise BadValueType('{!r} value {!r} not of type str'.format(
97                 self.name, self.value))
98
99         # Empty strings as environment variable values have different behavior
100         # on different operating systems. Just don't allow them.
101         if not self.allow_empty_values and self.value == '':
102             raise EmptyValue('{!r} value {!r} is the empty string'.format(
103                 self.name, self.value))
104
105         # Many tools have issues with newlines in environment variable values.
106         # Just don't allow them.
107         if '\n' in self.value:
108             raise NewlineInValue('{!r} value {!r} contains a newline'.format(
109                 self.name, self.value))
110
111         if not re.match(r'^[A-Z_][A-Z0-9_]*$', self.name, re.IGNORECASE):
112             raise BadVariableName('bad variable name {!r}'.format(self.name))
113
114     def unapply(self, env, orig_env):
115         if self.name in orig_env:
116             env[self.name] = orig_env[self.name]
117         else:
118             env.pop(self.name, None)
119
120
121 def _var_form(variable, windows=(os.name == 'nt')):
122     if windows:
123         return '%{}%'.format(variable)
124     return '${}'.format(variable)
125
126
127 class Set(_VariableAction):
128     """Set a variable."""
129     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
130         value = self.value
131         for var, replacement in replacements:
132             if var != self.name:
133                 value = value.replace(replacement, _var_form(var, windows))
134
135         if windows:
136             outs.write('set {name}={value}\n'.format(name=self.name,
137                                                      value=value))
138         else:
139             outs.write('{name}="{value}"\nexport {name}\n'.format(
140                 name=self.name, value=value))
141
142     def write_deactivate(self,
143                          outs,
144                          windows=(os.name == 'nt'),
145                          replacements=()):
146         del replacements  # Unused.
147
148         if windows:
149             outs.write('set {name}=\n'.format(name=self.name))
150         else:
151             outs.write('unset {name}\n'.format(name=self.name))
152
153     def apply(self, env):
154         env[self.name] = self.value
155
156     def json(self, data):
157         data['set'][self.name] = self.value
158
159
160 class Clear(_VariableAction):
161     """Remove a variable from the environment."""
162     def __init__(self, *args, **kwargs):
163         kwargs['value'] = ''
164         kwargs['allow_empty_values'] = True
165         super(Clear, self).__init__(*args, **kwargs)
166
167     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
168         del replacements  # Unused.
169         if windows:
170             outs.write('set {name}=\n'.format(**vars(self)))
171         else:
172             outs.write('unset {name}\n'.format(**vars(self)))
173
174     def apply(self, env):
175         if self.name in env:
176             del env[self.name]
177
178     def json(self, data):
179         data['set'][self.name] = None
180
181
182 def _initialize_path_like_variable(data, name):
183     default = {'append': [], 'prepend': [], 'remove': []}
184     data['modify'].setdefault(name, default)
185
186
187 def _remove_value_from_path(variable, value, pathsep):
188     return ('{variable}="$(echo "${variable}"'
189             ' | sed "s|{pathsep}{value}{pathsep}|{pathsep}|g;"'
190             ' | sed "s|^{value}{pathsep}||g;"'
191             ' | sed "s|{pathsep}{value}$||g;"'
192             ')"\nexport {variable}\n'.format(variable=variable,
193                                              value=value,
194                                              pathsep=pathsep))
195
196
197 class Remove(_VariableAction):
198     """Remove a value from a PATH-like variable."""
199     def __init__(self, name, value, pathsep, *args, **kwargs):
200         super(Remove, self).__init__(name, value, *args, **kwargs)
201         self._pathsep = pathsep
202
203     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
204         value = self.value
205         for var, replacement in replacements:
206             if var != self.name:
207                 value = value.replace(replacement, _var_form(var, windows))
208
209         if windows:
210             pass
211             # TODO(pwbug/231) This does not seem to be supported when value
212             # contains a %variable%. Disabling for now.
213             # outs.write(':: Remove\n::   {value}\n:: from\n::   {name}\n'
214             #            ':: before adding it back.\n'
215             #            'set {name}=%{name}:{value}{pathsep}=%\n'.format(
216             #              name=self.name, value=value, pathsep=self._pathsep))
217
218         else:
219             outs.write('# Remove \n#   {value}\n# from\n#   {value}\n# before '
220                        'adding it back.\n')
221             outs.write(_remove_value_from_path(self.name, value,
222                                                self._pathsep))
223
224     def apply(self, env):
225         env[self.name] = env[self.name].replace(
226             '{}{}'.format(self.value, self._pathsep), '')
227         env[self.name] = env[self.name].replace(
228             '{}{}'.format(self._pathsep, self.value), '')
229
230     def json(self, data):
231         _initialize_path_like_variable(data, self.name)
232         data['modify'][self.name]['remove'].append(self.value)
233         if self.value in data['modify'][self.name]['append']:
234             data['modify'][self.name]['append'].remove(self.value)
235         if self.value in data['modify'][self.name]['prepend']:
236             data['modify'][self.name]['prepend'].remove(self.value)
237
238
239 class BadVariableValue(ValueError):
240     pass
241
242
243 def _append_prepend_check(action):
244     if '=' in action.value:
245         raise BadVariableValue('"{}" contains "="'.format(action.value))
246
247
248 class Prepend(_VariableAction):
249     """Prepend a value to a PATH-like variable."""
250     def __init__(self, name, value, join, *args, **kwargs):
251         super(Prepend, self).__init__(name, value, *args, **kwargs)
252         self._join = join
253
254     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
255         value = self.value
256         for var, replacement in replacements:
257             if var != self.name:
258                 value = value.replace(replacement, _var_form(var, windows))
259         value = self._join(value, _var_form(self.name, windows))
260
261         if windows:
262             outs.write('set {name}={value}\n'.format(name=self.name,
263                                                      value=value))
264         else:
265             outs.write('{name}="{value}"\nexport {name}\n'.format(
266                 name=self.name, value=value))
267
268     def write_deactivate(self,
269                          outs,
270                          windows=(os.name == 'nt'),
271                          replacements=()):
272         value = self.value
273         for var, replacement in replacements:
274             if var != self.name:
275                 value = value.replace(replacement, _var_form(var, windows))
276
277         outs.write(
278             _remove_value_from_path(self.name, value, self._join.pathsep))
279
280     def apply(self, env):
281         env[self.name] = self._join(self.value, env.get(self.name, ''))
282
283     def _check(self):
284         super(Prepend, self)._check()
285         _append_prepend_check(self)
286
287     def json(self, data):
288         _initialize_path_like_variable(data, self.name)
289         data['modify'][self.name]['prepend'].append(self.value)
290         if self.value in data['modify'][self.name]['remove']:
291             data['modify'][self.name]['remove'].remove(self.value)
292
293
294 class Append(_VariableAction):
295     """Append a value to a PATH-like variable. (Uncommon, see Prepend.)"""
296     def __init__(self, name, value, join, *args, **kwargs):
297         super(Append, self).__init__(name, value, *args, **kwargs)
298         self._join = join
299
300     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
301         value = self.value
302         for var, repl_value in replacements:
303             if var != self.name:
304                 value = value.replace(repl_value, _var_form(var, windows))
305         value = self._join(_var_form(self.name, windows), value)
306
307         if windows:
308             outs.write('set {name}={value}\n'.format(name=self.name,
309                                                      value=value))
310         else:
311             outs.write('{name}="{value}"\nexport {name}\n'.format(
312                 name=self.name, value=value))
313
314     def write_deactivate(self,
315                          outs,
316                          windows=(os.name == 'nt'),
317                          replacements=()):
318         value = self.value
319         for var, replacement in replacements:
320             if var != self.name:
321                 value = value.replace(replacement, _var_form(var, windows))
322
323         outs.write(
324             _remove_value_from_path(self.name, value, self._join.pathsep))
325
326     def apply(self, env):
327         env[self.name] = self._join(env.get(self.name, ''), self.value)
328
329     def _check(self):
330         super(Append, self)._check()
331         _append_prepend_check(self)
332
333     def json(self, data):
334         _initialize_path_like_variable(data, self.name)
335         data['modify'][self.name]['append'].append(self.value)
336         if self.value in data['modify'][self.name]['remove']:
337             data['modify'][self.name]['remove'].remove(self.value)
338
339
340 class BadEchoValue(ValueError):
341     pass
342
343
344 class Echo(_Action):
345     """Echo a value to the terminal."""
346     def __init__(self, value, newline, *args, **kwargs):
347         # These values act funny on Windows.
348         if value.lower() in ('off', 'on'):
349             raise BadEchoValue(value)
350         super(Echo, self).__init__(*args, **kwargs)
351         self.value = value
352         self._newline = newline
353
354     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
355         del replacements  # Unused.
356         # POSIX shells parse arguments and pass to echo, but Windows seems to
357         # pass the command line as is without parsing, so quoting is wrong.
358         if windows:
359             if self._newline:
360                 if not self.value:
361                     outs.write('echo.\n')
362                 else:
363                     outs.write('echo {}\n'.format(self.value))
364             else:
365                 outs.write('<nul set /p="{}"\n'.format(self.value))
366         else:
367             # TODO(mohrr) use shlex.quote().
368             outs.write('if [ -z "${PW_ENVSETUP_QUIET:-}" ]; then\n')
369             if self._newline:
370                 outs.write('  echo "{}"\n'.format(self.value))
371             else:
372                 outs.write('  echo -n "{}"\n'.format(self.value))
373             outs.write('fi\n')
374
375     def apply(self, env):
376         pass
377
378
379 class Comment(_Action):
380     """Add a comment to the init script."""
381     def __init__(self, value, *args, **kwargs):
382         super(Comment, self).__init__(*args, **kwargs)
383         self.value = value
384
385     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
386         del replacements  # Unused.
387         comment_char = '::' if windows else '#'
388         for line in self.value.splitlines():
389             outs.write('{} {}\n'.format(comment_char, line))
390
391     def apply(self, env):
392         pass
393
394
395 class Command(_Action):
396     """Run a command."""
397     def __init__(self, command, *args, **kwargs):
398         exit_on_error = kwargs.pop('exit_on_error', True)
399         super(Command, self).__init__(*args, **kwargs)
400         assert isinstance(command, (list, tuple))
401         self.command = command
402         self.exit_on_error = exit_on_error
403
404     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
405         del replacements  # Unused.
406         # TODO(mohrr) use shlex.quote here?
407         outs.write('{}\n'.format(' '.join(self.command)))
408         if not self.exit_on_error:
409             return
410
411         if windows:
412             outs.write(
413                 'if %ERRORLEVEL% neq 0 goto {}\n'.format(_SCRIPT_END_LABEL))
414         else:
415             # Assume failing command produced relevant output.
416             outs.write('if [ "$?" -ne 0 ]; then\n  return 1\nfi\n')
417
418     def apply(self, env):
419         pass
420
421
422 class Doctor(Command):
423     def __init__(self, *args, **kwargs):
424         log_level = 'warn' if 'PW_ENVSETUP_QUIET' in os.environ else 'info'
425         super(Doctor, self).__init__(
426             command=['pw', '--no-banner', '--loglevel', log_level, 'doctor'],
427             *args,
428             **kwargs)
429
430     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
431         super_call = lambda: super(Doctor, self).write(
432             outs, windows=windows, replacements=replacements)
433
434         if windows:
435             outs.write('if "%PW_ACTIVATE_SKIP_CHECKS%"=="" (\n')
436             super_call()
437             outs.write(') else (\n')
438             outs.write('echo Skipping environment check because '
439                        'PW_ACTIVATE_SKIP_CHECKS is set\n')
440             outs.write(')\n')
441         else:
442             outs.write('if [ -z "$PW_ACTIVATE_SKIP_CHECKS" ]; then\n')
443             super_call()
444             outs.write('else\n')
445             outs.write('echo Skipping environment check because '
446                        'PW_ACTIVATE_SKIP_CHECKS is set\n')
447             outs.write('fi\n')
448
449
450 class BlankLine(_Action):
451     """Write a blank line to the init script."""
452     def write(  # pylint: disable=no-self-use
453         self,
454         outs,
455         windows=(os.name == 'nt'),
456         replacements=()):
457         del replacements, windows  # Unused.
458         outs.write('\n')
459
460     def apply(self, env):
461         pass
462
463
464 class Function(_Action):
465     def __init__(self, name, body, *args, **kwargs):
466         super(Function, self).__init__(*args, **kwargs)
467         self._name = name
468         self._body = body
469
470     def write(self, outs, windows=(os.name == 'nt'), replacements=()):
471         del replacements  # Unused.
472         if windows:
473             return
474
475         outs.write("""
476 {name}() {{
477 {body}
478 }}
479         """.strip().format(name=self._name, body=self._body))
480
481     def apply(self, env):
482         pass
483
484
485 class Hash(_Action):
486     def write(  # pylint: disable=no-self-use
487         self,
488         outs,
489         windows=(os.name == 'nt'),
490         replacements=()):
491         del replacements  # Unused.
492
493         if windows:
494             return
495
496         outs.write('''
497 # This should detect bash and zsh, which have a hash command that must be
498 # called to get it to forget past commands. Without forgetting past
499 # commands the $PATH changes we made may not be respected.
500 if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then
501   hash -r\n
502 fi
503 ''')
504
505     def apply(self, env):
506         pass
507
508
509 class Join(object):  # pylint: disable=useless-object-inheritance
510     def __init__(self, pathsep=os.pathsep):
511         self.pathsep = pathsep
512
513     def __call__(self, *args):
514         if len(args) == 1 and isinstance(args[0], (list, tuple)):
515             args = args[0]
516         return self.pathsep.join(args)
517
518
519 # TODO(mohrr) remove disable=useless-object-inheritance once in Python 3.
520 # pylint: disable=useless-object-inheritance
521 class Environment(object):
522     """Stores the environment changes necessary for Pigweed.
523
524     These changes can be accessed by writing them to a file for bash-like
525     shells to source or by using this as a context manager.
526     """
527     def __init__(self, *args, **kwargs):
528         pathsep = kwargs.pop('pathsep', os.pathsep)
529         windows = kwargs.pop('windows', os.name == 'nt')
530         allcaps = kwargs.pop('allcaps', windows)
531         super(Environment, self).__init__(*args, **kwargs)
532         self._actions = []
533         self._pathsep = pathsep
534         self._windows = windows
535         self._allcaps = allcaps
536         self._replacements = []
537         self._join = Join(pathsep)
538         self._finalized = False
539
540     def add_replacement(self, variable, value=None):
541         self._replacements.append((variable, value))
542
543     def normalize_key(self, name):
544         if self._allcaps:
545             try:
546                 return name.upper()
547             except AttributeError:
548                 # The _Action class has code to handle incorrect types, so
549                 # we just ignore this error here.
550                 pass
551         return name
552
553     # A newline is printed after each high-level operation. Top-level
554     # operations should not invoke each other (this is why _remove() exists).
555
556     def set(self, name, value):
557         """Set a variable."""
558         assert not self._finalized
559         name = self.normalize_key(name)
560         self._actions.append(Set(name, value))
561         self._blankline()
562
563     def clear(self, name):
564         """Remove a variable."""
565         assert not self._finalized
566         name = self.normalize_key(name)
567         self._actions.append(Clear(name))
568         self._blankline()
569
570     def _remove(self, name, value):
571         """Remove a value from a variable."""
572         assert not self._finalized
573         name = self.normalize_key(name)
574         if self.get(name, None):
575             self._actions.append(Remove(name, value, self._pathsep))
576
577     def remove(self, name, value):
578         """Remove a value from a PATH-like variable."""
579         assert not self._finalized
580         self._remove(name, value)
581         self._blankline()
582
583     def append(self, name, value):
584         """Add a value to a PATH-like variable. Rarely used, see prepend()."""
585         assert not self._finalized
586         name = self.normalize_key(name)
587         if self.get(name, None):
588             self._remove(name, value)
589             self._actions.append(Append(name, value, self._join))
590         else:
591             self._actions.append(Set(name, value))
592         self._blankline()
593
594     def prepend(self, name, value):
595         """Add a value to the beginning of a PATH-like variable."""
596         assert not self._finalized
597         name = self.normalize_key(name)
598         if self.get(name, None):
599             self._remove(name, value)
600             self._actions.append(Prepend(name, value, self._join))
601         else:
602             self._actions.append(Set(name, value))
603         self._blankline()
604
605     def echo(self, value='', newline=True):
606         """Echo a value to the terminal."""
607         # echo() deliberately ignores self._finalized.
608         self._actions.append(Echo(value, newline))
609         if value:
610             self._blankline()
611
612     def comment(self, comment):
613         """Add a comment to the init script."""
614         # comment() deliberately ignores self._finalized.
615         self._actions.append(Comment(comment))
616         self._blankline()
617
618     def command(self, command, exit_on_error=True):
619         """Run a command."""
620         # command() deliberately ignores self._finalized.
621         self._actions.append(Command(command, exit_on_error=exit_on_error))
622         self._blankline()
623
624     def doctor(self):
625         """Run 'pw doctor'."""
626         self._actions.append(Doctor())
627
628     def function(self, name, body):
629         """Define a function."""
630         assert not self._finalized
631         self._actions.append(Command(name, body))
632         self._blankline()
633
634     def _blankline(self):
635         self._actions.append(BlankLine())
636
637     def finalize(self):
638         """Run cleanup at the end of environment setup."""
639         assert not self._finalized
640         self._finalized = True
641         self._actions.append(Hash())
642         self._blankline()
643
644         if not self._windows:
645             buf = StringIO()
646             for action in self._actions:
647                 action.write_deactivate(buf, windows=self._windows)
648             self._actions.append(Function('_pw_deactivate', buf.getvalue()))
649             self._blankline()
650
651     def write(self, outs):
652         """Writes a shell init script to outs."""
653         if self._windows:
654             outs.write('@echo off\n')
655
656         # This is a tuple and not a dictionary because we don't need random
657         # access and order needs to be preserved.
658         replacements = tuple((key, self.get(key) if value is None else value)
659                              for key, value in self._replacements)
660
661         for action in self._actions:
662             action.write(outs,
663                          windows=self._windows,
664                          replacements=replacements)
665
666         if self._windows:
667             outs.write(':{}\n'.format(_SCRIPT_END_LABEL))
668
669     def json(self, outs):
670         data = {
671             'modify': {},
672             'set': {},
673         }
674
675         for action in self._actions:
676             action.json(data)
677
678         json.dump(data, outs, indent=4, separators=(',', ': '))
679         outs.write('\n')
680
681     def write_deactivate(self, outs):
682         if self._windows:
683             outs.write('@echo off\n')
684
685         for action in reversed(self._actions):
686             action.write_deactivate(outs,
687                                     windows=self._windows,
688                                     replacements=())
689
690     @contextlib.contextmanager
691     def __call__(self, export=True):
692         """Set environment as if this was written to a file and sourced.
693
694         Within this context os.environ is updated with the environment
695         defined by this object. If export is False, os.environ is not updated,
696         but in both cases the updated environment is yielded.
697
698         On exit, previous environment is restored. See contextlib documentation
699         for details on how this function is structured.
700
701         Args:
702           export(bool): modify the environment of the running process (and
703             thus, its subprocesses)
704
705         Yields the new environment object.
706         """
707         try:
708             if export:
709                 orig_env = os.environ.copy()
710                 env = os.environ
711             else:
712                 env = os.environ.copy()
713
714             for action in self._actions:
715                 action.apply(env)
716
717             yield env
718
719         finally:
720             if export:
721                 for action in self._actions:
722                     action.unapply(env=os.environ, orig_env=orig_env)
723
724     def get(self, key, default=None):
725         """Get the value of a variable within context of this object."""
726         key = self.normalize_key(key)
727         with self(export=False) as env:
728             return env.get(key, default)
729
730     def __getitem__(self, key):
731         """Get the value of a variable within context of this object."""
732         key = self.normalize_key(key)
733         with self(export=False) as env:
734             return env[key]