241 lines
5.9 KiB
Python
241 lines
5.9 KiB
Python
# Copyright 2011 Google Inc. All Rights Reserved.
|
|
|
|
__author__ = 'kbaclawski@google.com (Krystian Baclawski)'
|
|
|
|
import abc
|
|
import collections
|
|
import os.path
|
|
|
|
|
|
class Shell(object):
|
|
"""Class used to build a string representation of a shell command."""
|
|
|
|
def __init__(self, cmd, *args, **kwargs):
|
|
assert all(key in ['path', 'ignore_error'] for key in kwargs)
|
|
|
|
self._cmd = cmd
|
|
self._args = list(args)
|
|
self._path = kwargs.get('path', '')
|
|
self._ignore_error = bool(kwargs.get('ignore_error', False))
|
|
|
|
def __str__(self):
|
|
cmdline = [os.path.join(self._path, self._cmd)]
|
|
cmdline.extend(self._args)
|
|
|
|
cmd = ' '.join(cmdline)
|
|
|
|
if self._ignore_error:
|
|
cmd = '{ %s; true; }' % cmd
|
|
|
|
return cmd
|
|
|
|
def AddOption(self, option):
|
|
self._args.append(option)
|
|
|
|
|
|
class Wrapper(object):
|
|
"""Wraps a command with environment which gets cleaned up after execution."""
|
|
|
|
_counter = 1
|
|
|
|
def __init__(self, command, cwd=None, env=None, umask=None):
|
|
# @param cwd: temporary working directory
|
|
# @param env: dictionary of environment variables
|
|
self._command = command
|
|
self._prefix = Chain()
|
|
self._suffix = Chain()
|
|
|
|
if cwd:
|
|
self._prefix.append(Shell('pushd', cwd))
|
|
self._suffix.insert(0, Shell('popd'))
|
|
|
|
if env:
|
|
for env_var, value in env.items():
|
|
self._prefix.append(Shell('%s=%s' % (env_var, value)))
|
|
self._suffix.insert(0, Shell('unset', env_var))
|
|
|
|
if umask:
|
|
umask_save_var = 'OLD_UMASK_%d' % self.counter
|
|
|
|
self._prefix.append(Shell('%s=$(umask)' % umask_save_var))
|
|
self._prefix.append(Shell('umask', umask))
|
|
self._suffix.insert(0, Shell('umask', '$%s' % umask_save_var))
|
|
|
|
@property
|
|
def counter(self):
|
|
counter = self._counter
|
|
self._counter += 1
|
|
return counter
|
|
|
|
def __str__(self):
|
|
return str(Chain(self._prefix, self._command, self._suffix))
|
|
|
|
|
|
class AbstractCommandContainer(collections.MutableSequence):
|
|
"""Common base for all classes that behave like command container."""
|
|
|
|
def __init__(self, *commands):
|
|
self._commands = list(commands)
|
|
|
|
def __contains__(self, command):
|
|
return command in self._commands
|
|
|
|
def __iter__(self):
|
|
return iter(self._commands)
|
|
|
|
def __len__(self):
|
|
return len(self._commands)
|
|
|
|
def __getitem__(self, index):
|
|
return self._commands[index]
|
|
|
|
def __setitem__(self, index, command):
|
|
self._commands[index] = self._ValidateCommandType(command)
|
|
|
|
def __delitem__(self, index):
|
|
del self._commands[index]
|
|
|
|
def insert(self, index, command):
|
|
self._commands.insert(index, self._ValidateCommandType(command))
|
|
|
|
@abc.abstractmethod
|
|
def __str__(self):
|
|
pass
|
|
|
|
@abc.abstractproperty
|
|
def stored_types(self):
|
|
pass
|
|
|
|
def _ValidateCommandType(self, command):
|
|
if type(command) not in self.stored_types:
|
|
raise TypeError('Command cannot have %s type.' % type(command))
|
|
else:
|
|
return command
|
|
|
|
def _StringifyCommands(self):
|
|
cmds = []
|
|
|
|
for cmd in self:
|
|
if isinstance(cmd, AbstractCommandContainer) and len(cmd) > 1:
|
|
cmds.append('{ %s; }' % cmd)
|
|
else:
|
|
cmds.append(str(cmd))
|
|
|
|
return cmds
|
|
|
|
|
|
class Chain(AbstractCommandContainer):
|
|
"""Container that chains shell commands using (&&) shell operator."""
|
|
|
|
@property
|
|
def stored_types(self):
|
|
return [str, Shell, Chain, Pipe]
|
|
|
|
def __str__(self):
|
|
return ' && '.join(self._StringifyCommands())
|
|
|
|
|
|
class Pipe(AbstractCommandContainer):
|
|
"""Container that chains shell commands using pipe (|) operator."""
|
|
|
|
def __init__(self, *commands, **kwargs):
|
|
assert all(key in ['input', 'output'] for key in kwargs)
|
|
|
|
AbstractCommandContainer.__init__(self, *commands)
|
|
|
|
self._input = kwargs.get('input', None)
|
|
self._output = kwargs.get('output', None)
|
|
|
|
@property
|
|
def stored_types(self):
|
|
return [str, Shell]
|
|
|
|
def __str__(self):
|
|
pipe = self._StringifyCommands()
|
|
|
|
if self._input:
|
|
pipe.insert(str(Shell('cat', self._input), 0))
|
|
|
|
if self._output:
|
|
pipe.append(str(Shell('tee', self._output)))
|
|
|
|
return ' | '.join(pipe)
|
|
|
|
# TODO(kbaclawski): Unfortunately we don't have any policy describing which
|
|
# directories can or cannot be touched by a job. Thus, I cannot decide how to
|
|
# protect a system against commands that are considered to be dangerous (like
|
|
# RmTree("${HOME}")). AFAIK we'll have to execute some commands with root access
|
|
# (especially for ChromeOS related jobs, which involve chroot-ing), which is
|
|
# even more scary.
|
|
|
|
|
|
def Copy(*args, **kwargs):
|
|
assert all(key in ['to_dir', 'recursive'] for key in kwargs.keys())
|
|
|
|
options = []
|
|
|
|
if 'to_dir' in kwargs:
|
|
options.extend(['-t', kwargs['to_dir']])
|
|
|
|
if 'recursive' in kwargs:
|
|
options.append('-r')
|
|
|
|
options.extend(args)
|
|
|
|
return Shell('cp', *options)
|
|
|
|
|
|
def RemoteCopyFrom(from_machine, from_path, to_path, username=None):
|
|
from_path = os.path.expanduser(from_path) + '/'
|
|
to_path = os.path.expanduser(to_path) + '/'
|
|
|
|
if not username:
|
|
login = from_machine
|
|
else:
|
|
login = '%s@%s' % (username, from_machine)
|
|
|
|
return Chain(
|
|
MakeDir(to_path), Shell('rsync', '-a', '%s:%s' %
|
|
(login, from_path), to_path))
|
|
|
|
|
|
def MakeSymlink(to_path, link_name):
|
|
return Shell('ln', '-f', '-s', '-T', to_path, link_name)
|
|
|
|
|
|
def MakeDir(*dirs, **kwargs):
|
|
options = ['-p']
|
|
|
|
mode = kwargs.get('mode', None)
|
|
|
|
if mode:
|
|
options.extend(['-m', str(mode)])
|
|
|
|
options.extend(dirs)
|
|
|
|
return Shell('mkdir', *options)
|
|
|
|
|
|
def RmTree(*dirs):
|
|
return Shell('rm', '-r', '-f', *dirs)
|
|
|
|
|
|
def UnTar(tar_file, dest_dir):
|
|
return Chain(
|
|
MakeDir(dest_dir), Shell('tar', '-x', '-f', tar_file, '-C', dest_dir))
|
|
|
|
|
|
def Tar(tar_file, *args):
|
|
options = ['-c']
|
|
|
|
if tar_file.endswith('.tar.bz2'):
|
|
options.append('-j')
|
|
elif tar_file.endswith('.tar.gz'):
|
|
options.append('-z')
|
|
else:
|
|
assert tar_file.endswith('.tar')
|
|
|
|
options.extend(['-f', tar_file])
|
|
options.extend(args)
|
|
|
|
return Chain(MakeDir(os.path.dirname(tar_file)), Shell('tar', *options))
|