3289 lines
108 KiB
Python
3289 lines
108 KiB
Python
"""
|
|
KVM test utility functions.
|
|
|
|
@copyright: 2008-2009 Red Hat Inc.
|
|
"""
|
|
|
|
import time, string, random, socket, os, signal, re, logging, commands, cPickle
|
|
import fcntl, shelve, ConfigParser, threading, sys, UserDict, inspect, tarfile
|
|
import struct, shutil
|
|
from autotest_lib.client.bin import utils, os_dep
|
|
from autotest_lib.client.common_lib import error, logging_config
|
|
import rss_client, aexpect
|
|
try:
|
|
import koji
|
|
KOJI_INSTALLED = True
|
|
except ImportError:
|
|
KOJI_INSTALLED = False
|
|
|
|
# From include/linux/sockios.h
|
|
SIOCSIFHWADDR = 0x8924
|
|
SIOCGIFHWADDR = 0x8927
|
|
SIOCSIFFLAGS = 0x8914
|
|
SIOCGIFINDEX = 0x8933
|
|
SIOCBRADDIF = 0x89a2
|
|
# From linux/include/linux/if_tun.h
|
|
TUNSETIFF = 0x400454ca
|
|
TUNGETIFF = 0x800454d2
|
|
TUNGETFEATURES = 0x800454cf
|
|
IFF_UP = 0x1
|
|
IFF_TAP = 0x0002
|
|
IFF_NO_PI = 0x1000
|
|
IFF_VNET_HDR = 0x4000
|
|
|
|
def _lock_file(filename):
|
|
f = open(filename, "w")
|
|
fcntl.lockf(f, fcntl.LOCK_EX)
|
|
return f
|
|
|
|
|
|
def _unlock_file(f):
|
|
fcntl.lockf(f, fcntl.LOCK_UN)
|
|
f.close()
|
|
|
|
|
|
def is_vm(obj):
|
|
"""
|
|
Tests whether a given object is a VM object.
|
|
|
|
@param obj: Python object.
|
|
"""
|
|
return obj.__class__.__name__ == "VM"
|
|
|
|
|
|
class NetError(Exception):
|
|
pass
|
|
|
|
|
|
class TAPModuleError(NetError):
|
|
def __init__(self, devname, action="open", details=None):
|
|
NetError.__init__(self, devname)
|
|
self.devname = devname
|
|
self.details = details
|
|
|
|
def __str__(self):
|
|
e_msg = "Can't %s %s" % (self.action, self.devname)
|
|
if self.details is not None:
|
|
e_msg += " : %s" % self.details
|
|
return e_msg
|
|
|
|
|
|
class TAPNotExistError(NetError):
|
|
def __init__(self, ifname):
|
|
NetError.__init__(self, ifname)
|
|
self.ifname = ifname
|
|
|
|
def __str__(self):
|
|
return "Interface %s does not exist" % self.ifname
|
|
|
|
|
|
class TAPCreationError(NetError):
|
|
def __init__(self, ifname, details=None):
|
|
NetError.__init__(self, ifname, details)
|
|
self.ifname = ifname
|
|
self.details = details
|
|
|
|
def __str__(self):
|
|
e_msg = "Cannot create TAP device %s" % self.ifname
|
|
if self.details is not None:
|
|
e_msg += ": %s" % self.details
|
|
return e_msg
|
|
|
|
|
|
class TAPBringUpError(NetError):
|
|
def __init__(self, ifname):
|
|
NetError.__init__(self, ifname)
|
|
self.ifname = ifname
|
|
|
|
def __str__(self):
|
|
return "Cannot bring up TAP %s" % self.ifname
|
|
|
|
|
|
class BRAddIfError(NetError):
|
|
def __init__(self, ifname, brname, details):
|
|
NetError.__init__(self, ifname, brname, details)
|
|
self.ifname = ifname
|
|
self.brname = brname
|
|
self.details = details
|
|
|
|
def __str__(self):
|
|
return ("Can not add if %s to bridge %s: %s" %
|
|
(self.ifname, self.brname, self.details))
|
|
|
|
|
|
class HwAddrSetError(NetError):
|
|
def __init__(self, ifname, mac):
|
|
NetError.__init__(self, ifname, mac)
|
|
self.ifname = ifname
|
|
self.mac = mac
|
|
|
|
def __str__(self):
|
|
return "Can not set mac %s to interface %s" % (self.mac, self.ifname)
|
|
|
|
|
|
class HwAddrGetError(NetError):
|
|
def __init__(self, ifname):
|
|
NetError.__init__(self, ifname)
|
|
self.ifname = ifname
|
|
|
|
def __str__(self):
|
|
return "Can not get mac of interface %s" % self.ifname
|
|
|
|
|
|
class Env(UserDict.IterableUserDict):
|
|
"""
|
|
A dict-like object containing global objects used by tests.
|
|
"""
|
|
def __init__(self, filename=None, version=0):
|
|
"""
|
|
Create an empty Env object or load an existing one from a file.
|
|
|
|
If the version recorded in the file is lower than version, or if some
|
|
error occurs during unpickling, or if filename is not supplied,
|
|
create an empty Env object.
|
|
|
|
@param filename: Path to an env file.
|
|
@param version: Required env version (int).
|
|
"""
|
|
UserDict.IterableUserDict.__init__(self)
|
|
empty = {"version": version}
|
|
if filename:
|
|
self._filename = filename
|
|
try:
|
|
if os.path.isfile(filename):
|
|
f = open(filename, "r")
|
|
env = cPickle.load(f)
|
|
f.close()
|
|
if env.get("version", 0) >= version:
|
|
self.data = env
|
|
else:
|
|
logging.warning("Incompatible env file found. Not using it.")
|
|
self.data = empty
|
|
else:
|
|
# No previous env file found, proceed...
|
|
self.data = empty
|
|
# Almost any exception can be raised during unpickling, so let's
|
|
# catch them all
|
|
except Exception, e:
|
|
logging.warning(e)
|
|
self.data = empty
|
|
else:
|
|
self.data = empty
|
|
|
|
|
|
def save(self, filename=None):
|
|
"""
|
|
Pickle the contents of the Env object into a file.
|
|
|
|
@param filename: Filename to pickle the dict into. If not supplied,
|
|
use the filename from which the dict was loaded.
|
|
"""
|
|
filename = filename or self._filename
|
|
f = open(filename, "w")
|
|
cPickle.dump(self.data, f)
|
|
f.close()
|
|
|
|
|
|
def get_all_vms(self):
|
|
"""
|
|
Return a list of all VM objects in this Env object.
|
|
"""
|
|
return [o for o in self.values() if is_vm(o)]
|
|
|
|
|
|
def get_vm(self, name):
|
|
"""
|
|
Return a VM object by its name.
|
|
|
|
@param name: VM name.
|
|
"""
|
|
return self.get("vm__%s" % name)
|
|
|
|
|
|
def register_vm(self, name, vm):
|
|
"""
|
|
Register a VM in this Env object.
|
|
|
|
@param name: VM name.
|
|
@param vm: VM object.
|
|
"""
|
|
self["vm__%s" % name] = vm
|
|
|
|
|
|
def unregister_vm(self, name):
|
|
"""
|
|
Remove a given VM.
|
|
|
|
@param name: VM name.
|
|
"""
|
|
del self["vm__%s" % name]
|
|
|
|
|
|
def register_installer(self, installer):
|
|
"""
|
|
Register a installer that was just run
|
|
|
|
The installer will be available for other tests, so that
|
|
information about the installed KVM modules and qemu-kvm can be used by
|
|
them.
|
|
"""
|
|
self['last_installer'] = installer
|
|
|
|
|
|
def previous_installer(self):
|
|
"""
|
|
Return the last installer that was registered
|
|
"""
|
|
return self.get('last_installer')
|
|
|
|
|
|
class Params(UserDict.IterableUserDict):
|
|
"""
|
|
A dict-like object passed to every test.
|
|
"""
|
|
def objects(self, key):
|
|
"""
|
|
Return the names of objects defined using a given key.
|
|
|
|
@param key: The name of the key whose value lists the objects
|
|
(e.g. 'nics').
|
|
"""
|
|
return self.get(key, "").split()
|
|
|
|
|
|
def object_params(self, obj_name):
|
|
"""
|
|
Return a dict-like object containing the parameters of an individual
|
|
object.
|
|
|
|
This method behaves as follows: the suffix '_' + obj_name is removed
|
|
from all key names that have it. Other key names are left unchanged.
|
|
The values of keys with the suffix overwrite the values of their
|
|
suffixless versions.
|
|
|
|
@param obj_name: The name of the object (objects are listed by the
|
|
objects() method).
|
|
"""
|
|
suffix = "_" + obj_name
|
|
new_dict = self.copy()
|
|
for key in self:
|
|
if key.endswith(suffix):
|
|
new_key = key.split(suffix)[0]
|
|
new_dict[new_key] = self[key]
|
|
return new_dict
|
|
|
|
|
|
# Functions related to MAC/IP addresses
|
|
|
|
def _open_mac_pool(lock_mode):
|
|
lock_file = open("/tmp/mac_lock", "w+")
|
|
fcntl.lockf(lock_file, lock_mode)
|
|
pool = shelve.open("/tmp/address_pool")
|
|
return pool, lock_file
|
|
|
|
|
|
def _close_mac_pool(pool, lock_file):
|
|
pool.close()
|
|
fcntl.lockf(lock_file, fcntl.LOCK_UN)
|
|
lock_file.close()
|
|
|
|
|
|
def _generate_mac_address_prefix(mac_pool):
|
|
"""
|
|
Generate a random MAC address prefix and add it to the MAC pool dictionary.
|
|
If there's a MAC prefix there already, do not update the MAC pool and just
|
|
return what's in there. By convention we will set KVM autotest MAC
|
|
addresses to start with 0x9a.
|
|
|
|
@param mac_pool: The MAC address pool object.
|
|
@return: The MAC address prefix.
|
|
"""
|
|
if "prefix" in mac_pool:
|
|
prefix = mac_pool["prefix"]
|
|
else:
|
|
r = random.SystemRandom()
|
|
prefix = "9a:%02x:%02x:%02x:" % (r.randint(0x00, 0xff),
|
|
r.randint(0x00, 0xff),
|
|
r.randint(0x00, 0xff))
|
|
mac_pool["prefix"] = prefix
|
|
return prefix
|
|
|
|
|
|
def generate_mac_address(vm_instance, nic_index):
|
|
"""
|
|
Randomly generate a MAC address and add it to the MAC address pool.
|
|
|
|
Try to generate a MAC address based on a randomly generated MAC address
|
|
prefix and add it to a persistent dictionary.
|
|
key = VM instance + NIC index, value = MAC address
|
|
e.g. {'20100310-165222-Wt7l:0': '9a:5d:94:6a:9b:f9'}
|
|
|
|
@param vm_instance: The instance attribute of a VM.
|
|
@param nic_index: The index of the NIC.
|
|
@return: MAC address string.
|
|
"""
|
|
mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
|
|
key = "%s:%s" % (vm_instance, nic_index)
|
|
if key in mac_pool:
|
|
mac = mac_pool[key]
|
|
else:
|
|
prefix = _generate_mac_address_prefix(mac_pool)
|
|
r = random.SystemRandom()
|
|
while key not in mac_pool:
|
|
mac = prefix + "%02x:%02x" % (r.randint(0x00, 0xff),
|
|
r.randint(0x00, 0xff))
|
|
if mac in mac_pool.values():
|
|
continue
|
|
mac_pool[key] = mac
|
|
_close_mac_pool(mac_pool, lock_file)
|
|
return mac
|
|
|
|
|
|
def free_mac_address(vm_instance, nic_index):
|
|
"""
|
|
Remove a MAC address from the address pool.
|
|
|
|
@param vm_instance: The instance attribute of a VM.
|
|
@param nic_index: The index of the NIC.
|
|
"""
|
|
mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
|
|
key = "%s:%s" % (vm_instance, nic_index)
|
|
if key in mac_pool:
|
|
del mac_pool[key]
|
|
_close_mac_pool(mac_pool, lock_file)
|
|
|
|
|
|
def set_mac_address(vm_instance, nic_index, mac):
|
|
"""
|
|
Set a MAC address in the pool.
|
|
|
|
@param vm_instance: The instance attribute of a VM.
|
|
@param nic_index: The index of the NIC.
|
|
"""
|
|
mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_EX)
|
|
mac_pool["%s:%s" % (vm_instance, nic_index)] = mac
|
|
_close_mac_pool(mac_pool, lock_file)
|
|
|
|
|
|
def get_mac_address(vm_instance, nic_index):
|
|
"""
|
|
Return a MAC address from the pool.
|
|
|
|
@param vm_instance: The instance attribute of a VM.
|
|
@param nic_index: The index of the NIC.
|
|
@return: MAC address string.
|
|
"""
|
|
mac_pool, lock_file = _open_mac_pool(fcntl.LOCK_SH)
|
|
mac = mac_pool.get("%s:%s" % (vm_instance, nic_index))
|
|
_close_mac_pool(mac_pool, lock_file)
|
|
return mac
|
|
|
|
|
|
def verify_ip_address_ownership(ip, macs, timeout=10.0):
|
|
"""
|
|
Use arping and the ARP cache to make sure a given IP address belongs to one
|
|
of the given MAC addresses.
|
|
|
|
@param ip: An IP address.
|
|
@param macs: A list or tuple of MAC addresses.
|
|
@return: True iff ip is assigned to a MAC address in macs.
|
|
"""
|
|
# Compile a regex that matches the given IP address and any of the given
|
|
# MAC addresses
|
|
mac_regex = "|".join("(%s)" % mac for mac in macs)
|
|
regex = re.compile(r"\b%s\b.*\b(%s)\b" % (ip, mac_regex), re.IGNORECASE)
|
|
|
|
# Check the ARP cache
|
|
o = commands.getoutput("%s -n" % find_command("arp"))
|
|
if regex.search(o):
|
|
return True
|
|
|
|
# Get the name of the bridge device for arping
|
|
o = commands.getoutput("%s route get %s" % (find_command("ip"), ip))
|
|
dev = re.findall("dev\s+\S+", o, re.IGNORECASE)
|
|
if not dev:
|
|
return False
|
|
dev = dev[0].split()[-1]
|
|
|
|
# Send an ARP request
|
|
o = commands.getoutput("%s -f -c 3 -I %s %s" %
|
|
(find_command("arping"), dev, ip))
|
|
return bool(regex.search(o))
|
|
|
|
|
|
# Utility functions for dealing with external processes
|
|
|
|
def find_command(cmd):
|
|
for dir in ["/usr/local/sbin", "/usr/local/bin",
|
|
"/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
|
|
file = os.path.join(dir, cmd)
|
|
if os.path.exists(file):
|
|
return file
|
|
raise ValueError('Missing command: %s' % cmd)
|
|
|
|
|
|
def pid_exists(pid):
|
|
"""
|
|
Return True if a given PID exists.
|
|
|
|
@param pid: Process ID number.
|
|
"""
|
|
try:
|
|
os.kill(pid, 0)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
|
|
def safe_kill(pid, signal):
|
|
"""
|
|
Attempt to send a signal to a given process that may or may not exist.
|
|
|
|
@param signal: Signal number.
|
|
"""
|
|
try:
|
|
os.kill(pid, signal)
|
|
return True
|
|
except:
|
|
return False
|
|
|
|
|
|
def kill_process_tree(pid, sig=signal.SIGKILL):
|
|
"""Signal a process and all of its children.
|
|
|
|
If the process does not exist -- return.
|
|
|
|
@param pid: The pid of the process to signal.
|
|
@param sig: The signal to send to the processes.
|
|
"""
|
|
if not safe_kill(pid, signal.SIGSTOP):
|
|
return
|
|
children = commands.getoutput("ps --ppid=%d -o pid=" % pid).split()
|
|
for child in children:
|
|
kill_process_tree(int(child), sig)
|
|
safe_kill(pid, sig)
|
|
safe_kill(pid, signal.SIGCONT)
|
|
|
|
|
|
def get_git_branch(repository, branch, srcdir, commit=None, lbranch=None):
|
|
"""
|
|
Retrieves a given git code repository.
|
|
|
|
@param repository: Git repository URL
|
|
"""
|
|
logging.info("Fetching git [REP '%s' BRANCH '%s' COMMIT '%s'] -> %s",
|
|
repository, branch, commit, srcdir)
|
|
if not os.path.exists(srcdir):
|
|
os.makedirs(srcdir)
|
|
os.chdir(srcdir)
|
|
|
|
if os.path.exists(".git"):
|
|
utils.system("git reset --hard")
|
|
else:
|
|
utils.system("git init")
|
|
|
|
if not lbranch:
|
|
lbranch = branch
|
|
|
|
utils.system("git fetch -q -f -u -t %s %s:%s" %
|
|
(repository, branch, lbranch))
|
|
utils.system("git checkout %s" % lbranch)
|
|
if commit:
|
|
utils.system("git checkout %s" % commit)
|
|
|
|
h = utils.system_output('git log --pretty=format:"%H" -1')
|
|
try:
|
|
desc = "tag %s" % utils.system_output("git describe")
|
|
except error.CmdError:
|
|
desc = "no tag found"
|
|
|
|
logging.info("Commit hash for %s is %s (%s)", repository, h.strip(), desc)
|
|
return srcdir
|
|
|
|
|
|
def check_kvm_source_dir(source_dir):
|
|
"""
|
|
Inspects the kvm source directory and verifies its disposition. In some
|
|
occasions build may be dependant on the source directory disposition.
|
|
The reason why the return codes are numbers is that we might have more
|
|
changes on the source directory layout, so it's not scalable to just use
|
|
strings like 'old_repo', 'new_repo' and such.
|
|
|
|
@param source_dir: Source code path that will be inspected.
|
|
"""
|
|
os.chdir(source_dir)
|
|
has_qemu_dir = os.path.isdir('qemu')
|
|
has_kvm_dir = os.path.isdir('kvm')
|
|
if has_qemu_dir:
|
|
logging.debug("qemu directory detected, source dir layout 1")
|
|
return 1
|
|
if has_kvm_dir and not has_qemu_dir:
|
|
logging.debug("kvm directory detected, source dir layout 2")
|
|
return 2
|
|
else:
|
|
raise error.TestError("Unknown source dir layout, cannot proceed.")
|
|
|
|
|
|
# Functions and classes used for logging into guests and transferring files
|
|
|
|
class LoginError(Exception):
|
|
def __init__(self, msg, output):
|
|
Exception.__init__(self, msg, output)
|
|
self.msg = msg
|
|
self.output = output
|
|
|
|
def __str__(self):
|
|
return "%s (output: %r)" % (self.msg, self.output)
|
|
|
|
|
|
class LoginAuthenticationError(LoginError):
|
|
pass
|
|
|
|
|
|
class LoginTimeoutError(LoginError):
|
|
def __init__(self, output):
|
|
LoginError.__init__(self, "Login timeout expired", output)
|
|
|
|
|
|
class LoginProcessTerminatedError(LoginError):
|
|
def __init__(self, status, output):
|
|
LoginError.__init__(self, None, output)
|
|
self.status = status
|
|
|
|
def __str__(self):
|
|
return ("Client process terminated (status: %s, output: %r)" %
|
|
(self.status, self.output))
|
|
|
|
|
|
class LoginBadClientError(LoginError):
|
|
def __init__(self, client):
|
|
LoginError.__init__(self, None, None)
|
|
self.client = client
|
|
|
|
def __str__(self):
|
|
return "Unknown remote shell client: %r" % self.client
|
|
|
|
|
|
class SCPError(Exception):
|
|
def __init__(self, msg, output):
|
|
Exception.__init__(self, msg, output)
|
|
self.msg = msg
|
|
self.output = output
|
|
|
|
def __str__(self):
|
|
return "%s (output: %r)" % (self.msg, self.output)
|
|
|
|
|
|
class SCPAuthenticationError(SCPError):
|
|
pass
|
|
|
|
|
|
class SCPAuthenticationTimeoutError(SCPAuthenticationError):
|
|
def __init__(self, output):
|
|
SCPAuthenticationError.__init__(self, "Authentication timeout expired",
|
|
output)
|
|
|
|
|
|
class SCPTransferTimeoutError(SCPError):
|
|
def __init__(self, output):
|
|
SCPError.__init__(self, "Transfer timeout expired", output)
|
|
|
|
|
|
class SCPTransferFailedError(SCPError):
|
|
def __init__(self, status, output):
|
|
SCPError.__init__(self, None, output)
|
|
self.status = status
|
|
|
|
def __str__(self):
|
|
return ("SCP transfer failed (status: %s, output: %r)" %
|
|
(self.status, self.output))
|
|
|
|
|
|
def _remote_login(session, username, password, prompt, timeout=10):
|
|
"""
|
|
Log into a remote host (guest) using SSH or Telnet. Wait for questions
|
|
and provide answers. If timeout expires while waiting for output from the
|
|
child (e.g. a password prompt or a shell prompt) -- fail.
|
|
|
|
@brief: Log into a remote host (guest) using SSH or Telnet.
|
|
|
|
@param session: An Expect or ShellSession instance to operate on
|
|
@param username: The username to send in reply to a login prompt
|
|
@param password: The password to send in reply to a password prompt
|
|
@param prompt: The shell prompt that indicates a successful login
|
|
@param timeout: The maximal time duration (in seconds) to wait for each
|
|
step of the login procedure (i.e. the "Are you sure" prompt, the
|
|
password prompt, the shell prompt, etc)
|
|
@raise LoginTimeoutError: If timeout expires
|
|
@raise LoginAuthenticationError: If authentication fails
|
|
@raise LoginProcessTerminatedError: If the client terminates during login
|
|
@raise LoginError: If some other error occurs
|
|
"""
|
|
password_prompt_count = 0
|
|
login_prompt_count = 0
|
|
|
|
while True:
|
|
try:
|
|
match, text = session.read_until_last_line_matches(
|
|
[r"[Aa]re you sure", r"[Pp]assword:\s*$", r"[Ll]ogin:\s*$",
|
|
r"[Cc]onnection.*closed", r"[Cc]onnection.*refused",
|
|
r"[Pp]lease wait", r"[Ww]arning", prompt],
|
|
timeout=timeout, internal_timeout=0.5)
|
|
if match == 0: # "Are you sure you want to continue connecting"
|
|
logging.debug("Got 'Are you sure...', sending 'yes'")
|
|
session.sendline("yes")
|
|
continue
|
|
elif match == 1: # "password:"
|
|
if password_prompt_count == 0:
|
|
logging.debug("Got password prompt, sending '%s'", password)
|
|
session.sendline(password)
|
|
password_prompt_count += 1
|
|
continue
|
|
else:
|
|
raise LoginAuthenticationError("Got password prompt twice",
|
|
text)
|
|
elif match == 2: # "login:"
|
|
if login_prompt_count == 0 and password_prompt_count == 0:
|
|
logging.debug("Got username prompt; sending '%s'", username)
|
|
session.sendline(username)
|
|
login_prompt_count += 1
|
|
continue
|
|
else:
|
|
if login_prompt_count > 0:
|
|
msg = "Got username prompt twice"
|
|
else:
|
|
msg = "Got username prompt after password prompt"
|
|
raise LoginAuthenticationError(msg, text)
|
|
elif match == 3: # "Connection closed"
|
|
raise LoginError("Client said 'connection closed'", text)
|
|
elif match == 4: # "Connection refused"
|
|
raise LoginError("Client said 'connection refused'", text)
|
|
elif match == 5: # "Please wait"
|
|
logging.debug("Got 'Please wait'")
|
|
timeout = 30
|
|
continue
|
|
elif match == 6: # "Warning added RSA"
|
|
logging.debug("Got 'Warning added RSA to known host list")
|
|
continue
|
|
elif match == 7: # prompt
|
|
logging.debug("Got shell prompt -- logged in")
|
|
break
|
|
except aexpect.ExpectTimeoutError, e:
|
|
raise LoginTimeoutError(e.output)
|
|
except aexpect.ExpectProcessTerminatedError, e:
|
|
raise LoginProcessTerminatedError(e.status, e.output)
|
|
|
|
|
|
def remote_login(client, host, port, username, password, prompt, linesep="\n",
|
|
log_filename=None, timeout=10):
|
|
"""
|
|
Log into a remote host (guest) using SSH/Telnet/Netcat.
|
|
|
|
@param client: The client to use ('ssh', 'telnet' or 'nc')
|
|
@param host: Hostname or IP address
|
|
@param port: Port to connect to
|
|
@param username: Username (if required)
|
|
@param password: Password (if required)
|
|
@param prompt: Shell prompt (regular expression)
|
|
@param linesep: The line separator to use when sending lines
|
|
(e.g. '\\n' or '\\r\\n')
|
|
@param log_filename: If specified, log all output to this file
|
|
@param timeout: The maximal time duration (in seconds) to wait for
|
|
each step of the login procedure (i.e. the "Are you sure" prompt
|
|
or the password prompt)
|
|
@raise LoginBadClientError: If an unknown client is requested
|
|
@raise: Whatever _remote_login() raises
|
|
@return: A ShellSession object.
|
|
"""
|
|
if client == "ssh":
|
|
cmd = ("ssh -o UserKnownHostsFile=/dev/null "
|
|
"-o PreferredAuthentications=password -p %s %s@%s" %
|
|
(port, username, host))
|
|
elif client == "telnet":
|
|
cmd = "telnet -l %s %s %s" % (username, host, port)
|
|
elif client == "nc":
|
|
cmd = "nc %s %s" % (host, port)
|
|
else:
|
|
raise LoginBadClientError(client)
|
|
|
|
logging.debug("Trying to login with command '%s'", cmd)
|
|
session = aexpect.ShellSession(cmd, linesep=linesep, prompt=prompt)
|
|
try:
|
|
_remote_login(session, username, password, prompt, timeout)
|
|
except:
|
|
session.close()
|
|
raise
|
|
if log_filename:
|
|
session.set_output_func(log_line)
|
|
session.set_output_params((log_filename,))
|
|
return session
|
|
|
|
|
|
def wait_for_login(client, host, port, username, password, prompt, linesep="\n",
|
|
log_filename=None, timeout=240, internal_timeout=10):
|
|
"""
|
|
Make multiple attempts to log into a remote host (guest) until one succeeds
|
|
or timeout expires.
|
|
|
|
@param timeout: Total time duration to wait for a successful login
|
|
@param internal_timeout: The maximal time duration (in seconds) to wait for
|
|
each step of the login procedure (e.g. the "Are you sure" prompt
|
|
or the password prompt)
|
|
@see: remote_login()
|
|
@raise: Whatever remote_login() raises
|
|
@return: A ShellSession object.
|
|
"""
|
|
logging.debug("Attempting to log into %s:%s using %s (timeout %ds)",
|
|
host, port, client, timeout)
|
|
end_time = time.time() + timeout
|
|
while time.time() < end_time:
|
|
try:
|
|
return remote_login(client, host, port, username, password, prompt,
|
|
linesep, log_filename, internal_timeout)
|
|
except LoginError, e:
|
|
logging.debug(e)
|
|
time.sleep(2)
|
|
# Timeout expired; try one more time but don't catch exceptions
|
|
return remote_login(client, host, port, username, password, prompt,
|
|
linesep, log_filename, internal_timeout)
|
|
|
|
|
|
def _remote_scp(session, password_list, transfer_timeout=600, login_timeout=10):
|
|
"""
|
|
Transfer file(s) to a remote host (guest) using SCP. Wait for questions
|
|
and provide answers. If login_timeout expires while waiting for output
|
|
from the child (e.g. a password prompt), fail. If transfer_timeout expires
|
|
while waiting for the transfer to complete, fail.
|
|
|
|
@brief: Transfer files using SCP, given a command line.
|
|
|
|
@param session: An Expect or ShellSession instance to operate on
|
|
@param password_list: Password list to send in reply to the password prompt
|
|
@param transfer_timeout: The time duration (in seconds) to wait for the
|
|
transfer to complete.
|
|
@param login_timeout: The maximal time duration (in seconds) to wait for
|
|
each step of the login procedure (i.e. the "Are you sure" prompt or
|
|
the password prompt)
|
|
@raise SCPAuthenticationError: If authentication fails
|
|
@raise SCPTransferTimeoutError: If the transfer fails to complete in time
|
|
@raise SCPTransferFailedError: If the process terminates with a nonzero
|
|
exit code
|
|
@raise SCPError: If some other error occurs
|
|
"""
|
|
password_prompt_count = 0
|
|
timeout = login_timeout
|
|
authentication_done = False
|
|
|
|
scp_type = len(password_list)
|
|
|
|
while True:
|
|
try:
|
|
match, text = session.read_until_last_line_matches(
|
|
[r"[Aa]re you sure", r"[Pp]assword:\s*$", r"lost connection"],
|
|
timeout=timeout, internal_timeout=0.5)
|
|
if match == 0: # "Are you sure you want to continue connecting"
|
|
logging.debug("Got 'Are you sure...', sending 'yes'")
|
|
session.sendline("yes")
|
|
continue
|
|
elif match == 1: # "password:"
|
|
if password_prompt_count == 0:
|
|
logging.debug("Got password prompt, sending '%s'" %
|
|
password_list[password_prompt_count])
|
|
session.sendline(password_list[password_prompt_count])
|
|
password_prompt_count += 1
|
|
timeout = transfer_timeout
|
|
if scp_type == 1:
|
|
authentication_done = True
|
|
continue
|
|
elif password_prompt_count == 1 and scp_type == 2:
|
|
logging.debug("Got password prompt, sending '%s'" %
|
|
password_list[password_prompt_count])
|
|
session.sendline(password_list[password_prompt_count])
|
|
password_prompt_count += 1
|
|
timeout = transfer_timeout
|
|
authentication_done = True
|
|
continue
|
|
else:
|
|
raise SCPAuthenticationError("Got password prompt twice",
|
|
text)
|
|
elif match == 2: # "lost connection"
|
|
raise SCPError("SCP client said 'lost connection'", text)
|
|
except aexpect.ExpectTimeoutError, e:
|
|
if authentication_done:
|
|
raise SCPTransferTimeoutError(e.output)
|
|
else:
|
|
raise SCPAuthenticationTimeoutError(e.output)
|
|
except aexpect.ExpectProcessTerminatedError, e:
|
|
if e.status == 0:
|
|
logging.debug("SCP process terminated with status 0")
|
|
break
|
|
else:
|
|
raise SCPTransferFailedError(e.status, e.output)
|
|
|
|
|
|
def remote_scp(command, password_list, log_filename=None, transfer_timeout=600,
|
|
login_timeout=10):
|
|
"""
|
|
Transfer file(s) to a remote host (guest) using SCP.
|
|
|
|
@brief: Transfer files using SCP, given a command line.
|
|
|
|
@param command: The command to execute
|
|
(e.g. "scp -r foobar root@localhost:/tmp/").
|
|
@param password_list: Password list to send in reply to a password prompt.
|
|
@param log_filename: If specified, log all output to this file
|
|
@param transfer_timeout: The time duration (in seconds) to wait for the
|
|
transfer to complete.
|
|
@param login_timeout: The maximal time duration (in seconds) to wait for
|
|
each step of the login procedure (i.e. the "Are you sure" prompt
|
|
or the password prompt)
|
|
@raise: Whatever _remote_scp() raises
|
|
"""
|
|
logging.debug("Trying to SCP with command '%s', timeout %ss",
|
|
command, transfer_timeout)
|
|
if log_filename:
|
|
output_func = log_line
|
|
output_params = (log_filename,)
|
|
else:
|
|
output_func = None
|
|
output_params = ()
|
|
session = aexpect.Expect(command,
|
|
output_func=output_func,
|
|
output_params=output_params)
|
|
try:
|
|
_remote_scp(session, password_list, transfer_timeout, login_timeout)
|
|
finally:
|
|
session.close()
|
|
|
|
|
|
def scp_to_remote(host, port, username, password, local_path, remote_path,
|
|
log_filename=None, timeout=600):
|
|
"""
|
|
Copy files to a remote host (guest) through scp.
|
|
|
|
@param host: Hostname or IP address
|
|
@param username: Username (if required)
|
|
@param password: Password (if required)
|
|
@param local_path: Path on the local machine where we are copying from
|
|
@param remote_path: Path on the remote machine where we are copying to
|
|
@param log_filename: If specified, log all output to this file
|
|
@param timeout: The time duration (in seconds) to wait for the transfer
|
|
to complete.
|
|
@raise: Whatever remote_scp() raises
|
|
"""
|
|
command = ("scp -v -o UserKnownHostsFile=/dev/null "
|
|
"-o PreferredAuthentications=password -r -P %s %s %s@%s:%s" %
|
|
(port, local_path, username, host, remote_path))
|
|
password_list = []
|
|
password_list.append(password)
|
|
return remote_scp(command, password_list, log_filename, timeout)
|
|
|
|
|
|
|
|
def scp_from_remote(host, port, username, password, remote_path, local_path,
|
|
log_filename=None, timeout=600):
|
|
"""
|
|
Copy files from a remote host (guest).
|
|
|
|
@param host: Hostname or IP address
|
|
@param username: Username (if required)
|
|
@param password: Password (if required)
|
|
@param local_path: Path on the local machine where we are copying from
|
|
@param remote_path: Path on the remote machine where we are copying to
|
|
@param log_filename: If specified, log all output to this file
|
|
@param timeout: The time duration (in seconds) to wait for the transfer
|
|
to complete.
|
|
@raise: Whatever remote_scp() raises
|
|
"""
|
|
command = ("scp -v -o UserKnownHostsFile=/dev/null "
|
|
"-o PreferredAuthentications=password -r -P %s %s@%s:%s %s" %
|
|
(port, username, host, remote_path, local_path))
|
|
password_list = []
|
|
password_list.append(password)
|
|
remote_scp(command, password_list, log_filename, timeout)
|
|
|
|
|
|
def scp_between_remotes(src, dst, port, s_passwd, d_passwd, s_name, d_name,
|
|
s_path, d_path, log_filename=None, timeout=600):
|
|
"""
|
|
Copy files from a remote host (guest) to another remote host (guest).
|
|
|
|
@param src/dst: Hostname or IP address of src and dst
|
|
@param s_name/d_name: Username (if required)
|
|
@param s_passwd/d_passwd: Password (if required)
|
|
@param s_path/d_path: Path on the remote machine where we are copying
|
|
from/to
|
|
@param log_filename: If specified, log all output to this file
|
|
@param timeout: The time duration (in seconds) to wait for the transfer
|
|
to complete.
|
|
|
|
@return: True on success and False on failure.
|
|
"""
|
|
command = ("scp -v -o UserKnownHostsFile=/dev/null -o "
|
|
"PreferredAuthentications=password -r -P %s %s@%s:%s %s@%s:%s" %
|
|
(port, s_name, src, s_path, d_name, dst, d_path))
|
|
password_list = []
|
|
password_list.append(s_passwd)
|
|
password_list.append(d_passwd)
|
|
return remote_scp(command, password_list, log_filename, timeout)
|
|
|
|
|
|
def copy_files_to(address, client, username, password, port, local_path,
|
|
remote_path, log_filename=None, verbose=False, timeout=600):
|
|
"""
|
|
Copy files to a remote host (guest) using the selected client.
|
|
|
|
@param client: Type of transfer client
|
|
@param username: Username (if required)
|
|
@param password: Password (if requried)
|
|
@param local_path: Path on the local machine where we are copying from
|
|
@param remote_path: Path on the remote machine where we are copying to
|
|
@param address: Address of remote host(guest)
|
|
@param log_filename: If specified, log all output to this file (SCP only)
|
|
@param verbose: If True, log some stats using logging.debug (RSS only)
|
|
@param timeout: The time duration (in seconds) to wait for the transfer to
|
|
complete.
|
|
@raise: Whatever remote_scp() raises
|
|
"""
|
|
if client == "scp":
|
|
scp_to_remote(address, port, username, password, local_path,
|
|
remote_path, log_filename, timeout)
|
|
elif client == "rss":
|
|
log_func = None
|
|
if verbose:
|
|
log_func = logging.debug
|
|
c = rss_client.FileUploadClient(address, port, log_func)
|
|
c.upload(local_path, remote_path, timeout)
|
|
c.close()
|
|
|
|
|
|
def copy_files_from(address, client, username, password, port, remote_path,
|
|
local_path, log_filename=None, verbose=False, timeout=600):
|
|
"""
|
|
Copy files from a remote host (guest) using the selected client.
|
|
|
|
@param client: Type of transfer client
|
|
@param username: Username (if required)
|
|
@param password: Password (if requried)
|
|
@param remote_path: Path on the remote machine where we are copying from
|
|
@param local_path: Path on the local machine where we are copying to
|
|
@param address: Address of remote host(guest)
|
|
@param log_filename: If specified, log all output to this file (SCP only)
|
|
@param verbose: If True, log some stats using logging.debug (RSS only)
|
|
@param timeout: The time duration (in seconds) to wait for the transfer to
|
|
complete.
|
|
@raise: Whatever remote_scp() raises
|
|
"""
|
|
if client == "scp":
|
|
scp_from_remote(address, port, username, password, remote_path,
|
|
local_path, log_filename, timeout)
|
|
elif client == "rss":
|
|
log_func = None
|
|
if verbose:
|
|
log_func = logging.debug
|
|
c = rss_client.FileDownloadClient(address, port, log_func)
|
|
c.download(remote_path, local_path, timeout)
|
|
c.close()
|
|
|
|
|
|
# The following are utility functions related to ports.
|
|
|
|
def is_port_free(port, address):
|
|
"""
|
|
Return True if the given port is available for use.
|
|
|
|
@param port: Port number
|
|
"""
|
|
try:
|
|
s = socket.socket()
|
|
#s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
if address == "localhost":
|
|
s.bind(("localhost", port))
|
|
free = True
|
|
else:
|
|
s.connect((address, port))
|
|
free = False
|
|
except socket.error:
|
|
if address == "localhost":
|
|
free = False
|
|
else:
|
|
free = True
|
|
s.close()
|
|
return free
|
|
|
|
|
|
def find_free_port(start_port, end_port, address="localhost"):
|
|
"""
|
|
Return a host free port in the range [start_port, end_port].
|
|
|
|
@param start_port: First port that will be checked.
|
|
@param end_port: Port immediately after the last one that will be checked.
|
|
"""
|
|
for i in range(start_port, end_port):
|
|
if is_port_free(i, address):
|
|
return i
|
|
return None
|
|
|
|
|
|
def find_free_ports(start_port, end_port, count, address="localhost"):
|
|
"""
|
|
Return count of host free ports in the range [start_port, end_port].
|
|
|
|
@count: Initial number of ports known to be free in the range.
|
|
@param start_port: First port that will be checked.
|
|
@param end_port: Port immediately after the last one that will be checked.
|
|
"""
|
|
ports = []
|
|
i = start_port
|
|
while i < end_port and count > 0:
|
|
if is_port_free(i, address):
|
|
ports.append(i)
|
|
count -= 1
|
|
i += 1
|
|
return ports
|
|
|
|
|
|
# An easy way to log lines to files when the logging system can't be used
|
|
|
|
_open_log_files = {}
|
|
_log_file_dir = "/tmp"
|
|
|
|
|
|
def log_line(filename, line):
|
|
"""
|
|
Write a line to a file. '\n' is appended to the line.
|
|
|
|
@param filename: Path of file to write to, either absolute or relative to
|
|
the dir set by set_log_file_dir().
|
|
@param line: Line to write.
|
|
"""
|
|
global _open_log_files, _log_file_dir
|
|
if filename not in _open_log_files:
|
|
path = get_path(_log_file_dir, filename)
|
|
try:
|
|
os.makedirs(os.path.dirname(path))
|
|
except OSError:
|
|
pass
|
|
_open_log_files[filename] = open(path, "w")
|
|
timestr = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
_open_log_files[filename].write("%s: %s\n" % (timestr, line))
|
|
_open_log_files[filename].flush()
|
|
|
|
|
|
def set_log_file_dir(dir):
|
|
"""
|
|
Set the base directory for log files created by log_line().
|
|
|
|
@param dir: Directory for log files.
|
|
"""
|
|
global _log_file_dir
|
|
_log_file_dir = dir
|
|
|
|
|
|
# The following are miscellaneous utility functions.
|
|
|
|
def get_path(base_path, user_path):
|
|
"""
|
|
Translate a user specified path to a real path.
|
|
If user_path is relative, append it to base_path.
|
|
If user_path is absolute, return it as is.
|
|
|
|
@param base_path: The base path of relative user specified paths.
|
|
@param user_path: The user specified path.
|
|
"""
|
|
if os.path.isabs(user_path):
|
|
return user_path
|
|
else:
|
|
return os.path.join(base_path, user_path)
|
|
|
|
|
|
def generate_random_string(length):
|
|
"""
|
|
Return a random string using alphanumeric characters.
|
|
|
|
@length: length of the string that will be generated.
|
|
"""
|
|
r = random.SystemRandom()
|
|
str = ""
|
|
chars = string.letters + string.digits
|
|
while length > 0:
|
|
str += r.choice(chars)
|
|
length -= 1
|
|
return str
|
|
|
|
def generate_random_id():
|
|
"""
|
|
Return a random string suitable for use as a qemu id.
|
|
"""
|
|
return "id" + generate_random_string(6)
|
|
|
|
|
|
def generate_tmp_file_name(file, ext=None, dir='/tmp/'):
|
|
"""
|
|
Returns a temporary file name. The file is not created.
|
|
"""
|
|
while True:
|
|
file_name = (file + '-' + time.strftime("%Y%m%d-%H%M%S-") +
|
|
generate_random_string(4))
|
|
if ext:
|
|
file_name += '.' + ext
|
|
file_name = os.path.join(dir, file_name)
|
|
if not os.path.exists(file_name):
|
|
break
|
|
|
|
return file_name
|
|
|
|
|
|
def format_str_for_message(str):
|
|
"""
|
|
Format str so that it can be appended to a message.
|
|
If str consists of one line, prefix it with a space.
|
|
If str consists of multiple lines, prefix it with a newline.
|
|
|
|
@param str: string that will be formatted.
|
|
"""
|
|
lines = str.splitlines()
|
|
num_lines = len(lines)
|
|
str = "\n".join(lines)
|
|
if num_lines == 0:
|
|
return ""
|
|
elif num_lines == 1:
|
|
return " " + str
|
|
else:
|
|
return "\n" + str
|
|
|
|
|
|
def wait_for(func, timeout, first=0.0, step=1.0, text=None):
|
|
"""
|
|
If func() evaluates to True before timeout expires, return the
|
|
value of func(). Otherwise return None.
|
|
|
|
@brief: Wait until func() evaluates to True.
|
|
|
|
@param timeout: Timeout in seconds
|
|
@param first: Time to sleep before first attempt
|
|
@param steps: Time to sleep between attempts in seconds
|
|
@param text: Text to print while waiting, for debug purposes
|
|
"""
|
|
start_time = time.time()
|
|
end_time = time.time() + timeout
|
|
|
|
time.sleep(first)
|
|
|
|
while time.time() < end_time:
|
|
if text:
|
|
logging.debug("%s (%f secs)", text, (time.time() - start_time))
|
|
|
|
output = func()
|
|
if output:
|
|
return output
|
|
|
|
time.sleep(step)
|
|
|
|
return None
|
|
|
|
|
|
def get_hash_from_file(hash_path, dvd_basename):
|
|
"""
|
|
Get the a hash from a given DVD image from a hash file
|
|
(Hash files are usually named MD5SUM or SHA1SUM and are located inside the
|
|
download directories of the DVDs)
|
|
|
|
@param hash_path: Local path to a hash file.
|
|
@param cd_image: Basename of a CD image
|
|
"""
|
|
hash_file = open(hash_path, 'r')
|
|
for line in hash_file.readlines():
|
|
if dvd_basename in line:
|
|
return line.split()[0]
|
|
|
|
|
|
def run_tests(parser, job):
|
|
"""
|
|
Runs the sequence of KVM tests based on the list of dictionaries
|
|
generated by the configuration system, handling dependencies.
|
|
|
|
@param parser: Config parser object.
|
|
@param job: Autotest job object.
|
|
|
|
@return: True, if all tests ran passed, False if any of them failed.
|
|
"""
|
|
for i, d in enumerate(parser.get_dicts()):
|
|
logging.info("Test %4d: %s" % (i + 1, d["shortname"]))
|
|
|
|
status_dict = {}
|
|
failed = False
|
|
|
|
for dict in parser.get_dicts():
|
|
if dict.get("skip") == "yes":
|
|
continue
|
|
dependencies_satisfied = True
|
|
for dep in dict.get("dep"):
|
|
for test_name in status_dict.keys():
|
|
if not dep in test_name:
|
|
continue
|
|
# So the only really non-fatal state is WARN,
|
|
# All the others make it not safe to proceed with dependency
|
|
# execution
|
|
if status_dict[test_name] not in ['GOOD', 'WARN']:
|
|
dependencies_satisfied = False
|
|
break
|
|
test_iterations = int(dict.get("iterations", 1))
|
|
test_tag = dict.get("shortname")
|
|
|
|
if dependencies_satisfied:
|
|
# Setting up profilers during test execution.
|
|
profilers = dict.get("profilers", "").split()
|
|
for profiler in profilers:
|
|
job.profilers.add(profiler)
|
|
# We need only one execution, profiled, hence we're passing
|
|
# the profile_only parameter to job.run_test().
|
|
profile_only = bool(profilers) or None
|
|
current_status = job.run_test_detail(dict.get("vm_type"),
|
|
params=dict,
|
|
tag=test_tag,
|
|
iterations=test_iterations,
|
|
profile_only=profile_only)
|
|
for profiler in profilers:
|
|
job.profilers.delete(profiler)
|
|
else:
|
|
# We will force the test to fail as TestNA during preprocessing
|
|
dict['dependency_failed'] = 'yes'
|
|
current_status = job.run_test_detail(dict.get("vm_type"),
|
|
params=dict,
|
|
tag=test_tag,
|
|
iterations=test_iterations)
|
|
|
|
if not current_status:
|
|
failed = True
|
|
status_dict[dict.get("name")] = current_status
|
|
|
|
return not failed
|
|
|
|
|
|
def display_attributes(instance):
|
|
"""
|
|
Inspects a given class instance attributes and displays them, convenient
|
|
for debugging.
|
|
"""
|
|
logging.debug("Attributes set:")
|
|
for member in inspect.getmembers(instance):
|
|
name, value = member
|
|
attribute = getattr(instance, name)
|
|
if not (name.startswith("__") or callable(attribute) or not value):
|
|
logging.debug(" %s: %s", name, value)
|
|
|
|
|
|
def get_full_pci_id(pci_id):
|
|
"""
|
|
Get full PCI ID of pci_id.
|
|
|
|
@param pci_id: PCI ID of a device.
|
|
"""
|
|
cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id
|
|
status, full_id = commands.getstatusoutput(cmd)
|
|
if status != 0:
|
|
return None
|
|
return full_id
|
|
|
|
|
|
def get_vendor_from_pci_id(pci_id):
|
|
"""
|
|
Check out the device vendor ID according to pci_id.
|
|
|
|
@param pci_id: PCI ID of a device.
|
|
"""
|
|
cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id
|
|
return re.sub(":", " ", commands.getoutput(cmd))
|
|
|
|
|
|
def get_cpu_flags():
|
|
"""
|
|
Returns a list of the CPU flags
|
|
"""
|
|
flags_re = re.compile(r'^flags\s*:(.*)')
|
|
for line in open('/proc/cpuinfo').readlines():
|
|
match = flags_re.match(line)
|
|
if match:
|
|
return match.groups()[0].split()
|
|
return []
|
|
|
|
|
|
def get_cpu_vendor(cpu_flags=[], verbose=True):
|
|
"""
|
|
Returns the name of the CPU vendor, either intel, amd or unknown
|
|
"""
|
|
if not cpu_flags:
|
|
cpu_flags = get_cpu_flags()
|
|
|
|
if 'vmx' in cpu_flags:
|
|
vendor = 'intel'
|
|
elif 'svm' in cpu_flags:
|
|
vendor = 'amd'
|
|
else:
|
|
vendor = 'unknown'
|
|
|
|
if verbose:
|
|
logging.debug("Detected CPU vendor as '%s'", vendor)
|
|
return vendor
|
|
|
|
|
|
def get_archive_tarball_name(source_dir, tarball_name, compression):
|
|
'''
|
|
Get the name for a tarball file, based on source, name and compression
|
|
'''
|
|
if tarball_name is None:
|
|
tarball_name = os.path.basename(source_dir)
|
|
|
|
if not tarball_name.endswith('.tar'):
|
|
tarball_name = '%s.tar' % tarball_name
|
|
|
|
if compression and not tarball_name.endswith('.%s' % compression):
|
|
tarball_name = '%s.%s' % (tarball_name, compression)
|
|
|
|
return tarball_name
|
|
|
|
|
|
def archive_as_tarball(source_dir, dest_dir, tarball_name=None,
|
|
compression='bz2', verbose=True):
|
|
'''
|
|
Saves the given source directory to the given destination as a tarball
|
|
|
|
If the name of the archive is omitted, it will be taken from the
|
|
source_dir. If it is an absolute path, dest_dir will be ignored. But,
|
|
if both the destination directory and tarball anem is given, and the
|
|
latter is not an absolute path, they will be combined.
|
|
|
|
For archiving directory '/tmp' in '/net/server/backup' as file
|
|
'tmp.tar.bz2', simply use:
|
|
|
|
>>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup')
|
|
|
|
To save the file it with a different name, say 'host1-tmp.tar.bz2'
|
|
and save it under '/net/server/backup', use:
|
|
|
|
>>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup',
|
|
'host1-tmp')
|
|
|
|
To save with gzip compression instead (resulting in the file
|
|
'/net/server/backup/host1-tmp.tar.gz'), use:
|
|
|
|
>>> virt_utils.archive_as_tarball('/tmp', '/net/server/backup',
|
|
'host1-tmp', 'gz')
|
|
'''
|
|
tarball_name = get_archive_tarball_name(source_dir,
|
|
tarball_name,
|
|
compression)
|
|
if not os.path.isabs(tarball_name):
|
|
tarball_path = os.path.join(dest_dir, tarball_name)
|
|
else:
|
|
tarball_path = tarball_name
|
|
|
|
if verbose:
|
|
logging.debug('Archiving %s as %s' % (source_dir,
|
|
tarball_path))
|
|
|
|
os.chdir(os.path.dirname(source_dir))
|
|
tarball = tarfile.TarFile(name=tarball_path, mode='w')
|
|
tarball = tarball.open(name=tarball_path, mode='w:%s' % compression)
|
|
tarball.add(os.path.basename(source_dir))
|
|
tarball.close()
|
|
|
|
|
|
class Thread(threading.Thread):
|
|
"""
|
|
Run a function in a background thread.
|
|
"""
|
|
def __init__(self, target, args=(), kwargs={}):
|
|
"""
|
|
Initialize the instance.
|
|
|
|
@param target: Function to run in the thread.
|
|
@param args: Arguments to pass to target.
|
|
@param kwargs: Keyword arguments to pass to target.
|
|
"""
|
|
threading.Thread.__init__(self)
|
|
self._target = target
|
|
self._args = args
|
|
self._kwargs = kwargs
|
|
|
|
|
|
def run(self):
|
|
"""
|
|
Run target (passed to the constructor). No point in calling this
|
|
function directly. Call start() to make this function run in a new
|
|
thread.
|
|
"""
|
|
self._e = None
|
|
self._retval = None
|
|
try:
|
|
try:
|
|
self._retval = self._target(*self._args, **self._kwargs)
|
|
except:
|
|
self._e = sys.exc_info()
|
|
raise
|
|
finally:
|
|
# Avoid circular references (start() may be called only once so
|
|
# it's OK to delete these)
|
|
del self._target, self._args, self._kwargs
|
|
|
|
|
|
def join(self, timeout=None, suppress_exception=False):
|
|
"""
|
|
Join the thread. If target raised an exception, re-raise it.
|
|
Otherwise, return the value returned by target.
|
|
|
|
@param timeout: Timeout value to pass to threading.Thread.join().
|
|
@param suppress_exception: If True, don't re-raise the exception.
|
|
"""
|
|
threading.Thread.join(self, timeout)
|
|
try:
|
|
if self._e:
|
|
if not suppress_exception:
|
|
# Because the exception was raised in another thread, we
|
|
# need to explicitly insert the current context into it
|
|
s = error.exception_context(self._e[1])
|
|
s = error.join_contexts(error.get_context(), s)
|
|
error.set_exception_context(self._e[1], s)
|
|
raise self._e[0], self._e[1], self._e[2]
|
|
else:
|
|
return self._retval
|
|
finally:
|
|
# Avoid circular references (join() may be called multiple times
|
|
# so we can't delete these)
|
|
self._e = None
|
|
self._retval = None
|
|
|
|
|
|
def parallel(targets):
|
|
"""
|
|
Run multiple functions in parallel.
|
|
|
|
@param targets: A sequence of tuples or functions. If it's a sequence of
|
|
tuples, each tuple will be interpreted as (target, args, kwargs) or
|
|
(target, args) or (target,) depending on its length. If it's a
|
|
sequence of functions, the functions will be called without
|
|
arguments.
|
|
@return: A list of the values returned by the functions called.
|
|
"""
|
|
threads = []
|
|
for target in targets:
|
|
if isinstance(target, tuple) or isinstance(target, list):
|
|
t = Thread(*target)
|
|
else:
|
|
t = Thread(target)
|
|
threads.append(t)
|
|
t.start()
|
|
return [t.join() for t in threads]
|
|
|
|
|
|
class VirtLoggingConfig(logging_config.LoggingConfig):
|
|
"""
|
|
Used with the sole purpose of providing convenient logging setup
|
|
for the KVM test auxiliary programs.
|
|
"""
|
|
def configure_logging(self, results_dir=None, verbose=False):
|
|
super(VirtLoggingConfig, self).configure_logging(use_console=True,
|
|
verbose=verbose)
|
|
|
|
|
|
class PciAssignable(object):
|
|
"""
|
|
Request PCI assignable devices on host. It will check whether to request
|
|
PF (physical Functions) or VF (Virtual Functions).
|
|
"""
|
|
def __init__(self, type="vf", driver=None, driver_option=None,
|
|
names=None, devices_requested=None):
|
|
"""
|
|
Initialize parameter 'type' which could be:
|
|
vf: Virtual Functions
|
|
pf: Physical Function (actual hardware)
|
|
mixed: Both includes VFs and PFs
|
|
|
|
If pass through Physical NIC cards, we need to specify which devices
|
|
to be assigned, e.g. 'eth1 eth2'.
|
|
|
|
If pass through Virtual Functions, we need to specify how many vfs
|
|
are going to be assigned, e.g. passthrough_count = 8 and max_vfs in
|
|
config file.
|
|
|
|
@param type: PCI device type.
|
|
@param driver: Kernel module for the PCI assignable device.
|
|
@param driver_option: Module option to specify the maximum number of
|
|
VFs (eg 'max_vfs=7')
|
|
@param names: Physical NIC cards correspondent network interfaces,
|
|
e.g.'eth1 eth2 ...'
|
|
@param devices_requested: Number of devices being requested.
|
|
"""
|
|
self.type = type
|
|
self.driver = driver
|
|
self.driver_option = driver_option
|
|
if names:
|
|
self.name_list = names.split()
|
|
if devices_requested:
|
|
self.devices_requested = int(devices_requested)
|
|
else:
|
|
self.devices_requested = None
|
|
|
|
|
|
def _get_pf_pci_id(self, name, search_str):
|
|
"""
|
|
Get the PF PCI ID according to name.
|
|
|
|
@param name: Name of the PCI device.
|
|
@param search_str: Search string to be used on lspci.
|
|
"""
|
|
cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name
|
|
s, pci_id = commands.getstatusoutput(cmd)
|
|
if not (s or "Cannot get driver information" in pci_id):
|
|
return pci_id[5:]
|
|
cmd = "lspci | awk '/%s/ {print $1}'" % search_str
|
|
pci_ids = [id for id in commands.getoutput(cmd).splitlines()]
|
|
nic_id = int(re.search('[0-9]+', name).group(0))
|
|
if (len(pci_ids) - 1) < nic_id:
|
|
return None
|
|
return pci_ids[nic_id]
|
|
|
|
|
|
def _release_dev(self, pci_id):
|
|
"""
|
|
Release a single PCI device.
|
|
|
|
@param pci_id: PCI ID of a given PCI device.
|
|
"""
|
|
base_dir = "/sys/bus/pci"
|
|
full_id = get_full_pci_id(pci_id)
|
|
vendor_id = get_vendor_from_pci_id(pci_id)
|
|
drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id)
|
|
if 'pci-stub' in os.readlink(drv_path):
|
|
cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path)
|
|
if os.system(cmd):
|
|
return False
|
|
|
|
stub_path = os.path.join(base_dir, "drivers/pci-stub")
|
|
cmd = "echo '%s' > %s/unbind" % (full_id, stub_path)
|
|
if os.system(cmd):
|
|
return False
|
|
|
|
driver = self.dev_drivers[pci_id]
|
|
cmd = "echo '%s' > %s/bind" % (full_id, driver)
|
|
if os.system(cmd):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def get_vf_devs(self):
|
|
"""
|
|
Catch all VFs PCI IDs.
|
|
|
|
@return: List with all PCI IDs for the Virtual Functions avaliable
|
|
"""
|
|
if not self.sr_iov_setup():
|
|
return []
|
|
|
|
cmd = "lspci | awk '/Virtual Function/ {print $1}'"
|
|
return commands.getoutput(cmd).split()
|
|
|
|
|
|
def get_pf_devs(self):
|
|
"""
|
|
Catch all PFs PCI IDs.
|
|
|
|
@return: List with all PCI IDs for the physical hardware requested
|
|
"""
|
|
pf_ids = []
|
|
for name in self.name_list:
|
|
pf_id = self._get_pf_pci_id(name, "Ethernet")
|
|
if not pf_id:
|
|
continue
|
|
pf_ids.append(pf_id)
|
|
return pf_ids
|
|
|
|
|
|
def get_devs(self, count):
|
|
"""
|
|
Check out all devices' PCI IDs according to their name.
|
|
|
|
@param count: count number of PCI devices needed for pass through
|
|
@return: a list of all devices' PCI IDs
|
|
"""
|
|
if self.type == "vf":
|
|
vf_ids = self.get_vf_devs()
|
|
elif self.type == "pf":
|
|
vf_ids = self.get_pf_devs()
|
|
elif self.type == "mixed":
|
|
vf_ids = self.get_vf_devs()
|
|
vf_ids.extend(self.get_pf_devs())
|
|
return vf_ids[0:count]
|
|
|
|
|
|
def get_vfs_count(self):
|
|
"""
|
|
Get VFs count number according to lspci.
|
|
"""
|
|
# FIXME: Need to think out a method of identify which
|
|
# 'virtual function' belongs to which physical card considering
|
|
# that if the host has more than one 82576 card. PCI_ID?
|
|
cmd = "lspci | grep 'Virtual Function' | wc -l"
|
|
return int(commands.getoutput(cmd))
|
|
|
|
|
|
def check_vfs_count(self):
|
|
"""
|
|
Check VFs count number according to the parameter driver_options.
|
|
"""
|
|
# Network card 82576 has two network interfaces and each can be
|
|
# virtualized up to 7 virtual functions, therefore we multiply
|
|
# two for the value of driver_option 'max_vfs'.
|
|
expected_count = int((re.findall("(\d)", self.driver_option)[0])) * 2
|
|
return (self.get_vfs_count == expected_count)
|
|
|
|
|
|
def is_binded_to_stub(self, full_id):
|
|
"""
|
|
Verify whether the device with full_id is already binded to pci-stub.
|
|
|
|
@param full_id: Full ID for the given PCI device
|
|
"""
|
|
base_dir = "/sys/bus/pci"
|
|
stub_path = os.path.join(base_dir, "drivers/pci-stub")
|
|
if os.path.exists(os.path.join(stub_path, full_id)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def sr_iov_setup(self):
|
|
"""
|
|
Ensure the PCI device is working in sr_iov mode.
|
|
|
|
Check if the PCI hardware device drive is loaded with the appropriate,
|
|
parameters (number of VFs), and if it's not, perform setup.
|
|
|
|
@return: True, if the setup was completed successfuly, False otherwise.
|
|
"""
|
|
re_probe = False
|
|
s, o = commands.getstatusoutput('lsmod | grep %s' % self.driver)
|
|
if s:
|
|
re_probe = True
|
|
elif not self.check_vfs_count():
|
|
os.system("modprobe -r %s" % self.driver)
|
|
re_probe = True
|
|
else:
|
|
return True
|
|
|
|
# Re-probe driver with proper number of VFs
|
|
if re_probe:
|
|
cmd = "modprobe %s %s" % (self.driver, self.driver_option)
|
|
logging.info("Loading the driver '%s' with option '%s'",
|
|
self.driver, self.driver_option)
|
|
s, o = commands.getstatusoutput(cmd)
|
|
if s:
|
|
return False
|
|
return True
|
|
|
|
|
|
def request_devs(self):
|
|
"""
|
|
Implement setup process: unbind the PCI device and then bind it
|
|
to the pci-stub driver.
|
|
|
|
@return: a list of successfully requested devices' PCI IDs.
|
|
"""
|
|
base_dir = "/sys/bus/pci"
|
|
stub_path = os.path.join(base_dir, "drivers/pci-stub")
|
|
|
|
self.pci_ids = self.get_devs(self.devices_requested)
|
|
logging.debug("The following pci_ids were found: %s", self.pci_ids)
|
|
requested_pci_ids = []
|
|
self.dev_drivers = {}
|
|
|
|
# Setup all devices specified for assignment to guest
|
|
for pci_id in self.pci_ids:
|
|
full_id = get_full_pci_id(pci_id)
|
|
if not full_id:
|
|
continue
|
|
drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id)
|
|
dev_prev_driver = os.path.realpath(os.path.join(drv_path,
|
|
os.readlink(drv_path)))
|
|
self.dev_drivers[pci_id] = dev_prev_driver
|
|
|
|
# Judge whether the device driver has been binded to stub
|
|
if not self.is_binded_to_stub(full_id):
|
|
logging.debug("Binding device %s to stub", full_id)
|
|
vendor_id = get_vendor_from_pci_id(pci_id)
|
|
stub_new_id = os.path.join(stub_path, 'new_id')
|
|
unbind_dev = os.path.join(drv_path, 'unbind')
|
|
stub_bind = os.path.join(stub_path, 'bind')
|
|
|
|
info_write_to_files = [(vendor_id, stub_new_id),
|
|
(full_id, unbind_dev),
|
|
(full_id, stub_bind)]
|
|
|
|
for content, file in info_write_to_files:
|
|
try:
|
|
utils.open_write_close(file, content)
|
|
except IOError:
|
|
logging.debug("Failed to write %s to file %s", content,
|
|
file)
|
|
continue
|
|
|
|
if not self.is_binded_to_stub(full_id):
|
|
logging.error("Binding device %s to stub failed", pci_id)
|
|
continue
|
|
else:
|
|
logging.debug("Device %s already binded to stub", pci_id)
|
|
requested_pci_ids.append(pci_id)
|
|
self.pci_ids = requested_pci_ids
|
|
return self.pci_ids
|
|
|
|
|
|
def release_devs(self):
|
|
"""
|
|
Release all PCI devices currently assigned to VMs back to the
|
|
virtualization host.
|
|
"""
|
|
try:
|
|
for pci_id in self.dev_drivers:
|
|
if not self._release_dev(pci_id):
|
|
logging.error("Failed to release device %s to host", pci_id)
|
|
else:
|
|
logging.info("Released device %s successfully", pci_id)
|
|
except:
|
|
return
|
|
|
|
|
|
class KojiClient(object):
|
|
"""
|
|
Stablishes a connection with the build system, either koji or brew.
|
|
|
|
This class provides convenience methods to retrieve information on packages
|
|
and the packages themselves hosted on the build system. Packages should be
|
|
specified in the KojiPgkSpec syntax.
|
|
"""
|
|
|
|
CMD_LOOKUP_ORDER = ['/usr/bin/brew', '/usr/bin/koji' ]
|
|
|
|
CONFIG_MAP = {'/usr/bin/brew': '/etc/brewkoji.conf',
|
|
'/usr/bin/koji': '/etc/koji.conf'}
|
|
|
|
|
|
def __init__(self, cmd=None):
|
|
"""
|
|
Verifies whether the system has koji or brew installed, then loads
|
|
the configuration file that will be used to download the files.
|
|
|
|
@type cmd: string
|
|
@param cmd: Optional command name, either 'brew' or 'koji'. If not
|
|
set, get_default_command() is used and to look for
|
|
one of them.
|
|
@raise: ValueError
|
|
"""
|
|
if not KOJI_INSTALLED:
|
|
raise ValueError('No koji/brew installed on the machine')
|
|
|
|
# Instance variables used by many methods
|
|
self.command = None
|
|
self.config = None
|
|
self.config_options = {}
|
|
self.session = None
|
|
|
|
# Set koji command or get default
|
|
if cmd is None:
|
|
self.command = self.get_default_command()
|
|
else:
|
|
self.command = cmd
|
|
|
|
# Check koji command
|
|
if not self.is_command_valid():
|
|
raise ValueError('Koji command "%s" is not valid' % self.command)
|
|
|
|
# Assuming command is valid, set configuration file and read it
|
|
self.config = self.CONFIG_MAP[self.command]
|
|
self.read_config()
|
|
|
|
# Setup koji session
|
|
server_url = self.config_options['server']
|
|
session_options = self.get_session_options()
|
|
self.session = koji.ClientSession(server_url,
|
|
session_options)
|
|
|
|
|
|
def read_config(self, check_is_valid=True):
|
|
'''
|
|
Reads options from the Koji configuration file
|
|
|
|
By default it checks if the koji configuration is valid
|
|
|
|
@type check_valid: boolean
|
|
@param check_valid: whether to include a check on the configuration
|
|
@raises: ValueError
|
|
@returns: None
|
|
'''
|
|
if check_is_valid:
|
|
if not self.is_config_valid():
|
|
raise ValueError('Koji config "%s" is not valid' % self.config)
|
|
|
|
config = ConfigParser.ConfigParser()
|
|
config.read(self.config)
|
|
|
|
basename = os.path.basename(self.command)
|
|
for name, value in config.items(basename):
|
|
self.config_options[name] = value
|
|
|
|
|
|
def get_session_options(self):
|
|
'''
|
|
Filter only options necessary for setting up a cobbler client session
|
|
|
|
@returns: only the options used for session setup
|
|
'''
|
|
session_options = {}
|
|
for name, value in self.config_options.items():
|
|
if name in ('user', 'password', 'debug_xmlrpc', 'debug'):
|
|
session_options[name] = value
|
|
return session_options
|
|
|
|
|
|
def is_command_valid(self):
|
|
'''
|
|
Checks if the currently set koji command is valid
|
|
|
|
@returns: True or False
|
|
'''
|
|
koji_command_ok = True
|
|
|
|
if not os.path.isfile(self.command):
|
|
logging.error('Koji command "%s" is not a regular file',
|
|
self.command)
|
|
koji_command_ok = False
|
|
|
|
if not os.access(self.command, os.X_OK):
|
|
logging.warning('Koji command "%s" is not executable: this is '
|
|
'not fatal but indicates an unexpected situation',
|
|
self.command)
|
|
|
|
if not self.command in self.CONFIG_MAP.keys():
|
|
logging.error('Koji command "%s" does not have a configuration '
|
|
'file associated to it', self.command)
|
|
koji_command_ok = False
|
|
|
|
return koji_command_ok
|
|
|
|
|
|
def is_config_valid(self):
|
|
'''
|
|
Checks if the currently set koji configuration is valid
|
|
|
|
@returns: True or False
|
|
'''
|
|
koji_config_ok = True
|
|
|
|
if not os.path.isfile(self.config):
|
|
logging.error('Koji config "%s" is not a regular file', self.config)
|
|
koji_config_ok = False
|
|
|
|
if not os.access(self.config, os.R_OK):
|
|
logging.error('Koji config "%s" is not readable', self.config)
|
|
koji_config_ok = False
|
|
|
|
config = ConfigParser.ConfigParser()
|
|
config.read(self.config)
|
|
basename = os.path.basename(self.command)
|
|
if not config.has_section(basename):
|
|
logging.error('Koji configuration file "%s" does not have a '
|
|
'section "%s", named after the base name of the '
|
|
'currently set koji command "%s"', self.config,
|
|
basename, self.command)
|
|
koji_config_ok = False
|
|
|
|
return koji_config_ok
|
|
|
|
|
|
def get_default_command(self):
|
|
'''
|
|
Looks up for koji or brew "binaries" on the system
|
|
|
|
Systems with plain koji usually don't have a brew cmd, while systems
|
|
with koji, have *both* koji and brew utilities. So we look for brew
|
|
first, and if found, we consider that the system is configured for
|
|
brew. If not, we consider this is a system with plain koji.
|
|
|
|
@returns: either koji or brew command line executable path, or None
|
|
'''
|
|
koji_command = None
|
|
for command in self.CMD_LOOKUP_ORDER:
|
|
if os.path.isfile(command):
|
|
koji_command = command
|
|
break
|
|
else:
|
|
koji_command_basename = os.path.basename(koji_command)
|
|
try:
|
|
koji_command = os_dep.command(koji_command_basename)
|
|
break
|
|
except ValueError:
|
|
pass
|
|
return koji_command
|
|
|
|
|
|
def get_pkg_info(self, pkg):
|
|
'''
|
|
Returns information from Koji on the package
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: information about the package, as a KojiPkgSpec instance
|
|
|
|
@returns: information from Koji about the specified package
|
|
'''
|
|
info = {}
|
|
if pkg.build is not None:
|
|
info = self.session.getBuild(int(pkg.build))
|
|
elif pkg.tag is not None and pkg.package is not None:
|
|
builds = self.session.listTagged(pkg.tag,
|
|
latest=True,
|
|
inherit=True,
|
|
package=pkg.package)
|
|
if builds:
|
|
info = builds[0]
|
|
return info
|
|
|
|
|
|
def is_pkg_valid(self, pkg):
|
|
'''
|
|
Checks if this package is altogether valid on Koji
|
|
|
|
This verifies if the build or tag specified in the package
|
|
specification actually exist on the Koji server
|
|
|
|
@returns: True or False
|
|
'''
|
|
valid = True
|
|
if pkg.build:
|
|
if not self.is_pkg_spec_build_valid(pkg):
|
|
valid = False
|
|
elif pkg.tag:
|
|
if not self.is_pkg_spec_tag_valid(pkg):
|
|
valid = False
|
|
else:
|
|
valid = False
|
|
return valid
|
|
|
|
|
|
def is_pkg_spec_build_valid(self, pkg):
|
|
'''
|
|
Checks if build is valid on Koji
|
|
|
|
@param pkg: a Pkg instance
|
|
'''
|
|
if pkg.build is not None:
|
|
info = self.session.getBuild(int(pkg.build))
|
|
if info:
|
|
return True
|
|
return False
|
|
|
|
|
|
def is_pkg_spec_tag_valid(self, pkg):
|
|
'''
|
|
Checks if tag is valid on Koji
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
'''
|
|
if pkg.tag is not None:
|
|
tag = self.session.getTag(pkg.tag)
|
|
if tag:
|
|
return True
|
|
return False
|
|
|
|
|
|
def get_pkg_rpm_info(self, pkg, arch=None):
|
|
'''
|
|
Returns a list of infomation on the RPM packages found on koji
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
@type arch: string
|
|
@param arch: packages built for this architecture, but also including
|
|
architecture independent (noarch) packages
|
|
'''
|
|
if arch is None:
|
|
arch = utils.get_arch()
|
|
rpms = []
|
|
info = self.get_pkg_info(pkg)
|
|
if info:
|
|
rpms = self.session.listRPMs(buildID=info['id'],
|
|
arches=[arch, 'noarch'])
|
|
if pkg.subpackages:
|
|
rpms = [d for d in rpms if d['name'] in pkg.subpackages]
|
|
return rpms
|
|
|
|
|
|
def get_pkg_rpm_names(self, pkg, arch=None):
|
|
'''
|
|
Gets the names for the RPM packages specified in pkg
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
@type arch: string
|
|
@param arch: packages built for this architecture, but also including
|
|
architecture independent (noarch) packages
|
|
'''
|
|
if arch is None:
|
|
arch = utils.get_arch()
|
|
rpms = self.get_pkg_rpm_info(pkg, arch)
|
|
return [rpm['name'] for rpm in rpms]
|
|
|
|
|
|
def get_pkg_rpm_file_names(self, pkg, arch=None):
|
|
'''
|
|
Gets the file names for the RPM packages specified in pkg
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
@type arch: string
|
|
@param arch: packages built for this architecture, but also including
|
|
architecture independent (noarch) packages
|
|
'''
|
|
if arch is None:
|
|
arch = utils.get_arch()
|
|
rpm_names = []
|
|
rpms = self.get_pkg_rpm_info(pkg, arch)
|
|
for rpm in rpms:
|
|
arch_rpm_name = koji.pathinfo.rpm(rpm)
|
|
rpm_name = os.path.basename(arch_rpm_name)
|
|
rpm_names.append(rpm_name)
|
|
return rpm_names
|
|
|
|
|
|
def get_pkg_urls(self, pkg, arch=None):
|
|
'''
|
|
Gets the urls for the packages specified in pkg
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
@type arch: string
|
|
@param arch: packages built for this architecture, but also including
|
|
architecture independent (noarch) packages
|
|
'''
|
|
info = self.get_pkg_info(pkg)
|
|
rpms = self.get_pkg_rpm_info(pkg, arch)
|
|
rpm_urls = []
|
|
for rpm in rpms:
|
|
rpm_name = koji.pathinfo.rpm(rpm)
|
|
url = ("%s/%s/%s/%s/%s" % (self.config_options['pkgurl'],
|
|
info['package_name'],
|
|
info['version'], info['release'],
|
|
rpm_name))
|
|
rpm_urls.append(url)
|
|
return rpm_urls
|
|
|
|
|
|
def get_pkgs(self, pkg, dst_dir, arch=None):
|
|
'''
|
|
Download the packages
|
|
|
|
@type pkg: KojiPkgSpec
|
|
@param pkg: a package specification
|
|
@type dst_dir: string
|
|
@param dst_dir: the destination directory, where the downloaded
|
|
packages will be saved on
|
|
@type arch: string
|
|
@param arch: packages built for this architecture, but also including
|
|
architecture independent (noarch) packages
|
|
'''
|
|
rpm_urls = self.get_pkg_urls(pkg, arch)
|
|
for url in rpm_urls:
|
|
utils.get_file(url,
|
|
os.path.join(dst_dir, os.path.basename(url)))
|
|
|
|
|
|
DEFAULT_KOJI_TAG = None
|
|
def set_default_koji_tag(tag):
|
|
'''
|
|
Sets the default tag that will be used
|
|
'''
|
|
global DEFAULT_KOJI_TAG
|
|
DEFAULT_KOJI_TAG = tag
|
|
|
|
|
|
def get_default_koji_tag():
|
|
return DEFAULT_KOJI_TAG
|
|
|
|
|
|
class KojiPkgSpec(object):
|
|
'''
|
|
A package specification syntax parser for Koji
|
|
|
|
This holds information on either tag or build, and packages to be fetched
|
|
from koji and possibly installed (features external do this class).
|
|
|
|
New objects can be created either by providing information in the textual
|
|
format or by using the actual parameters for tag, build, package and sub-
|
|
packages. The textual format is useful for command line interfaces and
|
|
configuration files, while using parameters is better for using this in
|
|
a programatic fashion.
|
|
|
|
The following sets of examples are interchangeable. Specifying all packages
|
|
part of build number 1000:
|
|
|
|
>>> from kvm_utils import KojiPkgSpec
|
|
>>> pkg = KojiPkgSpec('1000')
|
|
|
|
>>> pkg = KojiPkgSpec(build=1000)
|
|
|
|
Specifying only a subset of packages of build number 1000:
|
|
|
|
>>> pkg = KojiPkgSpec('1000:kernel,kernel-devel')
|
|
|
|
>>> pkg = KojiPkgSpec(build=1000,
|
|
subpackages=['kernel', 'kernel-devel'])
|
|
|
|
Specifying the latest build for the 'kernel' package tagged with 'dist-f14':
|
|
|
|
>>> pkg = KojiPkgSpec('dist-f14:kernel')
|
|
|
|
>>> pkg = KojiPkgSpec(tag='dist-f14', package='kernel')
|
|
|
|
Specifying the 'kernel' package using the default tag:
|
|
|
|
>>> kvm_utils.set_default_koji_tag('dist-f14')
|
|
>>> pkg = KojiPkgSpec('kernel')
|
|
|
|
>>> pkg = KojiPkgSpec(package='kernel')
|
|
|
|
Specifying the 'kernel' package using the default tag:
|
|
|
|
>>> kvm_utils.set_default_koji_tag('dist-f14')
|
|
>>> pkg = KojiPkgSpec('kernel')
|
|
|
|
>>> pkg = KojiPkgSpec(package='kernel')
|
|
|
|
If you do not specify a default tag, and give a package name without an
|
|
explicit tag, your package specification is considered invalid:
|
|
|
|
>>> print kvm_utils.get_default_koji_tag()
|
|
None
|
|
>>> print kvm_utils.KojiPkgSpec('kernel').is_valid()
|
|
False
|
|
|
|
>>> print kvm_utils.KojiPkgSpec(package='kernel').is_valid()
|
|
False
|
|
'''
|
|
|
|
SEP = ':'
|
|
|
|
def __init__(self, text='', tag=None, build=None,
|
|
package=None, subpackages=[]):
|
|
'''
|
|
Instantiates a new KojiPkgSpec object
|
|
|
|
@type text: string
|
|
@param text: a textual representation of a package on Koji that
|
|
will be parsed
|
|
@type tag: string
|
|
@param tag: a koji tag, example: Fedora-14-RELEASE
|
|
(see U{http://fedoraproject.org/wiki/Koji#Tags_and_Targets})
|
|
@type build: number
|
|
@param build: a koji build, example: 1001
|
|
(see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture})
|
|
@type package: string
|
|
@param package: a koji package, example: python
|
|
(see U{http://fedoraproject.org/wiki/Koji#Koji_Architecture})
|
|
@type subpackages: list of strings
|
|
@param subpackages: a list of package names, usually a subset of
|
|
the RPM packages generated by a given build
|
|
'''
|
|
|
|
# Set to None to indicate 'not set' (and be able to use 'is')
|
|
self.tag = None
|
|
self.build = None
|
|
self.package = None
|
|
self.subpackages = []
|
|
|
|
self.default_tag = None
|
|
|
|
# Textual representation takes precedence (most common use case)
|
|
if text:
|
|
self.parse(text)
|
|
else:
|
|
self.tag = tag
|
|
self.build = build
|
|
self.package = package
|
|
self.subpackages = subpackages
|
|
|
|
# Set the default tag, if set, as a fallback
|
|
if not self.build and not self.tag:
|
|
default_tag = get_default_koji_tag()
|
|
if default_tag is not None:
|
|
self.tag = default_tag
|
|
|
|
|
|
def parse(self, text):
|
|
'''
|
|
Parses a textual representation of a package specification
|
|
|
|
@type text: string
|
|
@param text: textual representation of a package in koji
|
|
'''
|
|
parts = text.count(self.SEP) + 1
|
|
if parts == 1:
|
|
if text.isdigit():
|
|
self.build = text
|
|
else:
|
|
self.package = text
|
|
elif parts == 2:
|
|
part1, part2 = text.split(self.SEP)
|
|
if part1.isdigit():
|
|
self.build = part1
|
|
self.subpackages = part2.split(',')
|
|
else:
|
|
self.tag = part1
|
|
self.package = part2
|
|
elif parts >= 3:
|
|
# Instead of erroring on more arguments, we simply ignore them
|
|
# This makes the parser suitable for future syntax additions, such
|
|
# as specifying the package architecture
|
|
part1, part2, part3 = text.split(self.SEP)[0:3]
|
|
self.tag = part1
|
|
self.package = part2
|
|
self.subpackages = part3.split(',')
|
|
|
|
|
|
def _is_invalid_neither_tag_or_build(self):
|
|
'''
|
|
Checks if this package is invalid due to not having either a valid
|
|
tag or build set, that is, both are empty.
|
|
|
|
@returns: True if this is invalid and False if it's valid
|
|
'''
|
|
return (self.tag is None and self.build is None)
|
|
|
|
|
|
def _is_invalid_package_but_no_tag(self):
|
|
'''
|
|
Checks if this package is invalid due to having a package name set
|
|
but tag or build set, that is, both are empty.
|
|
|
|
@returns: True if this is invalid and False if it's valid
|
|
'''
|
|
return (self.package and not self.tag)
|
|
|
|
|
|
def _is_invalid_subpackages_but_no_main_package(self):
|
|
'''
|
|
Checks if this package is invalid due to having a tag set (this is Ok)
|
|
but specifying subpackage names without specifying the main package
|
|
name.
|
|
|
|
Specifying subpackages without a main package name is only valid when
|
|
a build is used instead of a tag.
|
|
|
|
@returns: True if this is invalid and False if it's valid
|
|
'''
|
|
return (self.tag and self.subpackages and not self.package)
|
|
|
|
|
|
def is_valid(self):
|
|
'''
|
|
Checks if this package specification is valid.
|
|
|
|
Being valid means that it has enough and not conflicting information.
|
|
It does not validate that the packages specified actually existe on
|
|
the Koji server.
|
|
|
|
@returns: True or False
|
|
'''
|
|
if self._is_invalid_neither_tag_or_build():
|
|
return False
|
|
elif self._is_invalid_package_but_no_tag():
|
|
return False
|
|
elif self._is_invalid_subpackages_but_no_main_package():
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def describe_invalid(self):
|
|
'''
|
|
Describes why this is not valid, in a human friendly way
|
|
'''
|
|
if self._is_invalid_neither_tag_or_build():
|
|
return 'neither a tag or build are set, and of them should be set'
|
|
elif self._is_invalid_package_but_no_tag():
|
|
return 'package name specified but no tag is set'
|
|
elif self._is_invalid_subpackages_but_no_main_package():
|
|
return 'subpackages specified but no main package is set'
|
|
|
|
return 'unkwown reason, seems to be valid'
|
|
|
|
|
|
def describe(self):
|
|
'''
|
|
Describe this package specification, in a human friendly way
|
|
|
|
@returns: package specification description
|
|
'''
|
|
if self.is_valid():
|
|
description = ''
|
|
if not self.subpackages:
|
|
description += 'all subpackages from %s ' % self.package
|
|
else:
|
|
description += ('only subpackage(s) %s from package %s ' %
|
|
(', '.join(self.subpackages), self.package))
|
|
|
|
if self.build:
|
|
description += 'from build %s' % self.build
|
|
elif self.tag:
|
|
description += 'tagged with %s' % self.tag
|
|
else:
|
|
raise ValueError, 'neither build or tag is set'
|
|
|
|
return description
|
|
else:
|
|
return ('Invalid package specification: %s' %
|
|
self.describe_invalid())
|
|
|
|
|
|
def __repr__(self):
|
|
return ("<KojiPkgSpec tag=%s build=%s pkg=%s subpkgs=%s>" %
|
|
(self.tag, self.build, self.package,
|
|
", ".join(self.subpackages)))
|
|
|
|
|
|
def umount(src, mount_point, type):
|
|
"""
|
|
Umount the src mounted in mount_point.
|
|
|
|
@src: mount source
|
|
@mount_point: mount point
|
|
@type: file system type
|
|
"""
|
|
|
|
mount_string = "%s %s %s" % (src, mount_point, type)
|
|
if mount_string in file("/etc/mtab").read():
|
|
umount_cmd = "umount %s" % mount_point
|
|
try:
|
|
utils.system(umount_cmd)
|
|
return True
|
|
except error.CmdError:
|
|
return False
|
|
else:
|
|
logging.debug("%s is not mounted under %s", src, mount_point)
|
|
return True
|
|
|
|
|
|
def mount(src, mount_point, type, perm="rw"):
|
|
"""
|
|
Mount the src into mount_point of the host.
|
|
|
|
@src: mount source
|
|
@mount_point: mount point
|
|
@type: file system type
|
|
@perm: mount premission
|
|
"""
|
|
umount(src, mount_point, type)
|
|
mount_string = "%s %s %s %s" % (src, mount_point, type, perm)
|
|
|
|
if mount_string in file("/etc/mtab").read():
|
|
logging.debug("%s is already mounted in %s with %s",
|
|
src, mount_point, perm)
|
|
return True
|
|
|
|
mount_cmd = "mount -t %s %s %s -o %s" % (type, src, mount_point, perm)
|
|
try:
|
|
utils.system(mount_cmd)
|
|
except error.CmdError:
|
|
return False
|
|
|
|
logging.debug("Verify the mount through /etc/mtab")
|
|
if mount_string in file("/etc/mtab").read():
|
|
logging.debug("%s is successfully mounted", src)
|
|
return True
|
|
else:
|
|
logging.error("Can't find mounted NFS share - /etc/mtab contents \n%s",
|
|
file("/etc/mtab").read())
|
|
return False
|
|
|
|
|
|
class GitRepoHelper(object):
|
|
'''
|
|
Helps to deal with git repos, mostly fetching content from a repo
|
|
'''
|
|
def __init__(self, uri, branch, destination_dir, commit=None, lbranch=None):
|
|
'''
|
|
Instantiates a new GitRepoHelper
|
|
|
|
@type uri: string
|
|
@param uri: git repository url
|
|
@type branch: string
|
|
@param branch: git remote branch
|
|
@type destination_dir: string
|
|
@param destination_dir: path of a dir where to save downloaded code
|
|
@type commit: string
|
|
@param commit: specific commit to download
|
|
@type lbranch: string
|
|
@param lbranch: git local branch name, if different from remote
|
|
'''
|
|
self.uri = uri
|
|
self.branch = branch
|
|
self.destination_dir = destination_dir
|
|
self.commit = commit
|
|
if lbranch is None:
|
|
self.lbranch = branch
|
|
|
|
|
|
def init(self):
|
|
'''
|
|
Initializes a directory for receiving a verbatim copy of git repo
|
|
|
|
This creates a directory if necessary, and either resets or inits
|
|
the repo
|
|
'''
|
|
if not os.path.exists(self.destination_dir):
|
|
logging.debug('Creating directory %s for git repo %s',
|
|
self.destination_dir, self.uri)
|
|
os.makedirs(self.destination_dir)
|
|
|
|
os.chdir(self.destination_dir)
|
|
|
|
if os.path.exists('.git'):
|
|
logging.debug('Resetting previously existing git repo at %s for '
|
|
'receiving git repo %s',
|
|
self.destination_dir, self.uri)
|
|
utils.system('git reset --hard')
|
|
else:
|
|
logging.debug('Initializing new git repo at %s for receiving '
|
|
'git repo %s',
|
|
self.destination_dir, self.uri)
|
|
utils.system('git init')
|
|
|
|
|
|
def fetch(self):
|
|
'''
|
|
Performs a git fetch from the remote repo
|
|
'''
|
|
logging.info("Fetching git [REP '%s' BRANCH '%s'] -> %s",
|
|
self.uri, self.branch, self.destination_dir)
|
|
os.chdir(self.destination_dir)
|
|
utils.system("git fetch -q -f -u -t %s %s:%s" % (self.uri,
|
|
self.branch,
|
|
self.lbranch))
|
|
|
|
|
|
def checkout(self):
|
|
'''
|
|
Performs a git checkout for a given branch and start point (commit)
|
|
'''
|
|
os.chdir(self.destination_dir)
|
|
|
|
logging.debug('Checking out local branch %s', self.lbranch)
|
|
utils.system("git checkout %s" % self.lbranch)
|
|
|
|
if self.commit is not None:
|
|
logging.debug('Checking out commit %s', self.commit)
|
|
utils.system("git checkout %s" % self.commit)
|
|
|
|
h = utils.system_output('git log --pretty=format:"%H" -1').strip()
|
|
try:
|
|
desc = "tag %s" % utils.system_output("git describe")
|
|
except error.CmdError:
|
|
desc = "no tag found"
|
|
|
|
logging.info("Commit hash for %s is %s (%s)", self.name, h, desc)
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Performs all steps necessary to initialize and download a git repo
|
|
|
|
This includes the init, fetch and checkout steps in one single
|
|
utility method.
|
|
'''
|
|
self.init()
|
|
self.fetch()
|
|
self.checkout()
|
|
|
|
|
|
class GitRepoParamHelper(GitRepoHelper):
|
|
'''
|
|
Helps to deal with git repos specified in cartersian config files
|
|
|
|
This class attempts to make it simple to manage a git repo, by using a
|
|
naming standard that follows this basic syntax:
|
|
|
|
<prefix>_name_<suffix>
|
|
|
|
<prefix> is always 'git_repo' and <suffix> sets options for this git repo.
|
|
Example for repo named foo:
|
|
|
|
git_repo_foo_uri = git://git.foo.org/foo.git
|
|
git_repo_foo_branch = master
|
|
git_repo_foo_lbranch = master
|
|
git_repo_foo_commit = bb5fb8e678aabe286e74c4f2993dc2a9e550b627
|
|
'''
|
|
def __init__(self, params, name, destination_dir):
|
|
'''
|
|
Instantiates a new GitRepoParamHelper
|
|
'''
|
|
self.params = params
|
|
self.name = name
|
|
self.destination_dir = destination_dir
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to this repo
|
|
|
|
This method currently does everything that the parent class __init__()
|
|
method does, that is, sets all instance variables needed by other
|
|
methods. That means it's not strictly necessary to call parent's
|
|
__init__().
|
|
'''
|
|
config_prefix = 'git_repo_%s' % self.name
|
|
logging.debug('Parsing parameters for git repo %s, configuration '
|
|
'prefix is %s' % (self.name, config_prefix))
|
|
|
|
self.uri = self.params.get('%s_uri' % config_prefix)
|
|
logging.debug('Git repo %s uri: %s' % (self.name, self.uri))
|
|
|
|
self.branch = self.params.get('%s_branch' % config_prefix, 'master')
|
|
logging.debug('Git repo %s branch: %s' % (self.name, self.branch))
|
|
|
|
self.lbranch = self.params.get('%s_lbranch' % config_prefix)
|
|
if self.lbranch is None:
|
|
self.lbranch = self.branch
|
|
logging.debug('Git repo %s lbranch: %s' % (self.name, self.lbranch))
|
|
|
|
self.commit = self.params.get('%s_commit' % config_prefix)
|
|
if self.commit is None:
|
|
logging.debug('Git repo %s commit is not set' % self.name)
|
|
else:
|
|
logging.debug('Git repo %s commit: %s' % (self.name, self.commit))
|
|
|
|
|
|
class LocalSourceDirHelper(object):
|
|
'''
|
|
Helper class to deal with source code sitting somewhere in the filesystem
|
|
'''
|
|
def __init__(self, source_dir, destination_dir):
|
|
'''
|
|
@param source_dir:
|
|
@param destination_dir:
|
|
@return: new LocalSourceDirHelper instance
|
|
'''
|
|
self.source = source_dir
|
|
self.destination = destination_dir
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Copies the source directory to the destination directory
|
|
'''
|
|
if os.path.isdir(self.destination):
|
|
shutil.rmtree(self.destination)
|
|
|
|
if os.path.isdir(self.source):
|
|
shutil.copytree(self.source, self.destination)
|
|
|
|
|
|
class LocalSourceDirParamHelper(LocalSourceDirHelper):
|
|
'''
|
|
Helps to deal with source dirs specified in cartersian config files
|
|
|
|
This class attempts to make it simple to manage a source dir, by using a
|
|
naming standard that follows this basic syntax:
|
|
|
|
<prefix>_name_<suffix>
|
|
|
|
<prefix> is always 'local_src' and <suffix> sets options for this source
|
|
dir. Example for source dir named foo:
|
|
|
|
local_src_foo_path = /home/user/foo
|
|
'''
|
|
def __init__(self, params, name, destination_dir):
|
|
'''
|
|
Instantiate a new LocalSourceDirParamHelper
|
|
'''
|
|
self.params = params
|
|
self.name = name
|
|
self.destination_dir = destination_dir
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to source dir
|
|
'''
|
|
config_prefix = 'local_src_%s' % self.name
|
|
logging.debug('Parsing parameters for local source %s, configuration '
|
|
'prefix is %s' % (self.name, config_prefix))
|
|
|
|
self.path = self.params.get('%s_path' % config_prefix)
|
|
logging.debug('Local source directory %s path: %s' % (self.name,
|
|
self.path))
|
|
self.source = self.path
|
|
self.destination = self.destination_dir
|
|
|
|
|
|
class LocalTarHelper(object):
|
|
'''
|
|
Helper class to deal with source code in a local tarball
|
|
'''
|
|
def __init__(self, source, destination_dir):
|
|
self.source = source
|
|
self.destination = destination_dir
|
|
|
|
|
|
def extract(self):
|
|
'''
|
|
Extracts the tarball into the destination directory
|
|
'''
|
|
if os.path.isdir(self.destination):
|
|
shutil.rmtree(self.destination)
|
|
|
|
if os.path.isfile(self.source) and tarfile.is_tarfile(self.source):
|
|
|
|
name = os.path.basename(self.destination)
|
|
temp_dir = os.path.join(os.path.dirname(self.destination),
|
|
'%s.tmp' % name)
|
|
logging.debug('Temporary directory for extracting tarball is %s' %
|
|
temp_dir)
|
|
|
|
if not os.path.isdir(temp_dir):
|
|
os.makedirs(temp_dir)
|
|
|
|
tarball = tarfile.open(self.source)
|
|
tarball.extractall(temp_dir)
|
|
|
|
#
|
|
# If there's a directory at the toplevel of the tarfile, assume
|
|
# it's the root for the contents, usually source code
|
|
#
|
|
tarball_info = tarball.members[0]
|
|
if tarball_info.isdir():
|
|
content_path = os.path.join(temp_dir,
|
|
tarball_info.name)
|
|
else:
|
|
content_path = temp_dir
|
|
|
|
#
|
|
# Now move the content directory to the final destination
|
|
#
|
|
shutil.move(content_path, self.destination)
|
|
|
|
else:
|
|
raise OSError("%s is not a file or tar file" % self.source)
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Executes all action this helper is suposed to perform
|
|
|
|
This is the main entry point method for this class, and all other
|
|
helper classes.
|
|
'''
|
|
self.extract()
|
|
|
|
|
|
class LocalTarParamHelper(LocalTarHelper):
|
|
'''
|
|
Helps to deal with source tarballs specified in cartersian config files
|
|
|
|
This class attempts to make it simple to manage a tarball with source code,
|
|
by using a naming standard that follows this basic syntax:
|
|
|
|
<prefix>_name_<suffix>
|
|
|
|
<prefix> is always 'local_tar' and <suffix> sets options for this source
|
|
tarball. Example for source tarball named foo:
|
|
|
|
local_tar_foo_path = /tmp/foo-1.0.tar.gz
|
|
'''
|
|
def __init__(self, params, name, destination_dir):
|
|
'''
|
|
Instantiates a new LocalTarParamHelper
|
|
'''
|
|
self.params = params
|
|
self.name = name
|
|
self.destination_dir = destination_dir
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to this local tar helper
|
|
'''
|
|
config_prefix = 'local_tar_%s' % self.name
|
|
logging.debug('Parsing parameters for local tar %s, configuration '
|
|
'prefix is %s' % (self.name, config_prefix))
|
|
|
|
self.path = self.params.get('%s_path' % config_prefix)
|
|
logging.debug('Local source tar %s path: %s' % (self.name,
|
|
self.path))
|
|
self.source = self.path
|
|
self.destination = self.destination_dir
|
|
|
|
|
|
class RemoteTarHelper(LocalTarHelper):
|
|
'''
|
|
Helper that fetches a tarball and extracts it locally
|
|
'''
|
|
def __init__(self, source_uri, destination_dir):
|
|
self.source = source_uri
|
|
self.destination = destination_dir
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Executes all action this helper class is suposed to perform
|
|
|
|
This is the main entry point method for this class, and all other
|
|
helper classes.
|
|
|
|
This implementation fetches the remote tar file and then extracts
|
|
it using the functionality present in the parent class.
|
|
'''
|
|
name = os.path.basename(self.source)
|
|
base_dest = os.path.dirname(self.destination_dir)
|
|
dest = os.path.join(base_dest, name)
|
|
utils.get_file(self.source, dest)
|
|
self.source = dest
|
|
self.extract()
|
|
|
|
|
|
class RemoteTarParamHelper(RemoteTarHelper):
|
|
'''
|
|
Helps to deal with remote source tarballs specified in cartersian config
|
|
|
|
This class attempts to make it simple to manage a tarball with source code,
|
|
by using a naming standard that follows this basic syntax:
|
|
|
|
<prefix>_name_<suffix>
|
|
|
|
<prefix> is always 'local_tar' and <suffix> sets options for this source
|
|
tarball. Example for source tarball named foo:
|
|
|
|
remote_tar_foo_uri = http://foo.org/foo-1.0.tar.gz
|
|
'''
|
|
def __init__(self, params, name, destination_dir):
|
|
'''
|
|
Instantiates a new RemoteTarParamHelper instance
|
|
'''
|
|
self.params = params
|
|
self.name = name
|
|
self.destination_dir = destination_dir
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to this remote tar helper
|
|
'''
|
|
config_prefix = 'remote_tar_%s' % self.name
|
|
logging.debug('Parsing parameters for remote tar %s, configuration '
|
|
'prefix is %s' % (self.name, config_prefix))
|
|
|
|
self.uri = self.params.get('%s_uri' % config_prefix)
|
|
logging.debug('Remote source tar %s uri: %s' % (self.name,
|
|
self.uri))
|
|
self.source = self.uri
|
|
self.destination = self.destination_dir
|
|
|
|
|
|
class PatchHelper(object):
|
|
'''
|
|
Helper that encapsulates the patching of source code with patch files
|
|
'''
|
|
def __init__(self, source_dir, patches):
|
|
'''
|
|
Initializes a new PatchHelper
|
|
'''
|
|
self.source_dir = source_dir
|
|
self.patches = patches
|
|
|
|
|
|
def download(self):
|
|
'''
|
|
Copies patch files from remote locations to the source directory
|
|
'''
|
|
for patch in self.patches:
|
|
utils.get_file(patch, os.path.join(self.source_dir,
|
|
os.path.basename(patch)))
|
|
|
|
|
|
def patch(self):
|
|
'''
|
|
Patches the source dir with all patch files
|
|
'''
|
|
os.chdir(self.source_dir)
|
|
for patch in self.patches:
|
|
patch_file = os.path.join(self.source_dir,
|
|
os.path.basename(patch))
|
|
utils.system('patch -p1 < %s' % os.path.basename(patch))
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Performs all steps necessary to download patches and apply them
|
|
'''
|
|
self.download()
|
|
self.patch()
|
|
|
|
|
|
class PatchParamHelper(PatchHelper):
|
|
'''
|
|
Helps to deal with patches specified in cartersian config files
|
|
|
|
This class attempts to make it simple to patch source coude, by using a
|
|
naming standard that follows this basic syntax:
|
|
|
|
[<git_repo>|<local_src>|<local_tar>|<remote_tar>]_<name>_patches
|
|
|
|
<prefix> is either a 'local_src' or 'git_repo', that, together with <name>
|
|
specify a directory containing source code to receive the patches. That is,
|
|
for source code coming from git repo foo, patches would be specified as:
|
|
|
|
git_repo_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
|
|
|
|
And for for patches to be applied on local source code named also foo:
|
|
|
|
local_src_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
|
|
'''
|
|
def __init__(self, params, prefix, source_dir):
|
|
'''
|
|
Initializes a new PatchParamHelper instance
|
|
'''
|
|
self.params = params
|
|
self.prefix = prefix
|
|
self.source_dir = source_dir
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to this set of patches
|
|
|
|
This method currently does everything that the parent class __init__()
|
|
method does, that is, sets all instance variables needed by other
|
|
methods. That means it's not strictly necessary to call parent's
|
|
__init__().
|
|
'''
|
|
logging.debug('Parsing patch parameters for prefix %s' % self.prefix)
|
|
patches_param_key = '%s_patches' % self.prefix
|
|
|
|
self.patches_str = self.params.get(patches_param_key, '[]')
|
|
logging.debug('Patches config for prefix %s: %s' % (self.prefix,
|
|
self.patches_str))
|
|
|
|
self.patches = eval(self.patches_str)
|
|
logging.debug('Patches for prefix %s: %s' % (self.prefix,
|
|
", ".join(self.patches)))
|
|
|
|
|
|
class GnuSourceBuildInvalidSource(Exception):
|
|
'''
|
|
Exception raised when build source dir/file is not valid
|
|
'''
|
|
pass
|
|
|
|
|
|
class GnuSourceBuildHelper(object):
|
|
'''
|
|
Handles software installation of GNU-like source code
|
|
|
|
This basically means that the build will go though the classic GNU
|
|
autotools steps: ./configure, make, make install
|
|
'''
|
|
def __init__(self, source, build_dir, prefix,
|
|
configure_options=[]):
|
|
'''
|
|
@type source: string
|
|
@param source: source directory or tarball
|
|
@type prefix: string
|
|
@param prefix: installation prefix
|
|
@type build_dir: string
|
|
@param build_dir: temporary directory used for building the source code
|
|
@type configure_options: list
|
|
@param configure_options: options to pass to configure
|
|
@throws: GnuSourceBuildInvalidSource
|
|
'''
|
|
self.source = source
|
|
self.build_dir = build_dir
|
|
self.prefix = prefix
|
|
self.configure_options = configure_options
|
|
self.include_pkg_config_path()
|
|
|
|
|
|
def include_pkg_config_path(self):
|
|
'''
|
|
Adds the current prefix to the list of paths that pkg-config searches
|
|
|
|
This is currently not optional as there is no observed adverse side
|
|
effects of enabling this. As the "prefix" is usually only valid during
|
|
a test run, we believe that having other pkg-config files (*.pc) in
|
|
either '<prefix>/share/pkgconfig' or '<prefix>/lib/pkgconfig' is
|
|
exactly for the purpose of using them.
|
|
|
|
@returns: None
|
|
'''
|
|
env_var = 'PKG_CONFIG_PATH'
|
|
|
|
include_paths = [os.path.join(self.prefix, 'share', 'pkgconfig'),
|
|
os.path.join(self.prefix, 'lib', 'pkgconfig')]
|
|
|
|
if os.environ.has_key(env_var):
|
|
paths = os.environ[env_var].split(':')
|
|
for include_path in include_paths:
|
|
if include_path not in paths:
|
|
paths.append(include_path)
|
|
os.environ[env_var] = ':'.join(paths)
|
|
else:
|
|
os.environ[env_var] = ':'.join(include_paths)
|
|
|
|
logging.debug('PKG_CONFIG_PATH is: %s' % os.environ['PKG_CONFIG_PATH'])
|
|
|
|
|
|
def get_configure_path(self):
|
|
'''
|
|
Checks if 'configure' exists, if not, return 'autogen.sh' as a fallback
|
|
'''
|
|
configure_path = os.path.abspath(os.path.join(self.source,
|
|
"configure"))
|
|
autogen_path = os.path.abspath(os.path.join(self.source,
|
|
"autogen.sh"))
|
|
if os.path.exists(configure_path):
|
|
return configure_path
|
|
elif os.path.exists(autogen_path):
|
|
return autogen_path
|
|
else:
|
|
raise GnuSourceBuildInvalidSource('configure script does not exist')
|
|
|
|
|
|
def get_available_configure_options(self):
|
|
'''
|
|
Return the list of available options of a GNU like configure script
|
|
|
|
This will run the "configure" script at the source directory
|
|
|
|
@returns: list of options accepted by configure script
|
|
'''
|
|
help_raw = utils.system_output('%s --help' % self.get_configure_path(),
|
|
ignore_status=True)
|
|
help_output = help_raw.split("\n")
|
|
option_list = []
|
|
for line in help_output:
|
|
cleaned_line = line.lstrip()
|
|
if cleaned_line.startswith("--"):
|
|
option = cleaned_line.split()[0]
|
|
option = option.split("=")[0]
|
|
option_list.append(option)
|
|
|
|
return option_list
|
|
|
|
|
|
def enable_debug_symbols(self):
|
|
'''
|
|
Enables option that leaves debug symbols on compiled software
|
|
|
|
This makes debugging a lot easier.
|
|
'''
|
|
enable_debug_option = "--disable-strip"
|
|
if enable_debug_option in self.get_available_configure_options():
|
|
self.configure_options.append(enable_debug_option)
|
|
logging.debug('Enabling debug symbols with option: %s' %
|
|
enable_debug_option)
|
|
|
|
|
|
def get_configure_command(self):
|
|
'''
|
|
Formats configure script with all options set
|
|
|
|
@returns: string with all configure options, including prefix
|
|
'''
|
|
prefix_option = "--prefix=%s" % self.prefix
|
|
options = self.configure_options
|
|
options.append(prefix_option)
|
|
return "%s %s" % (self.get_configure_path(),
|
|
" ".join(options))
|
|
|
|
|
|
def configure(self):
|
|
'''
|
|
Runs the "configure" script passing apropriate command line options
|
|
'''
|
|
configure_command = self.get_configure_command()
|
|
logging.info('Running configure on build dir')
|
|
os.chdir(self.build_dir)
|
|
utils.system(configure_command)
|
|
|
|
|
|
def make(self):
|
|
'''
|
|
Runs "make" using the correct number of parallel jobs
|
|
'''
|
|
parallel_make_jobs = utils.count_cpus()
|
|
make_command = "make -j %s" % parallel_make_jobs
|
|
logging.info("Running make on build dir")
|
|
os.chdir(self.build_dir)
|
|
utils.system(make_command)
|
|
|
|
|
|
def make_install(self):
|
|
'''
|
|
Runs "make install"
|
|
'''
|
|
os.chdir(self.build_dir)
|
|
utils.system("make install")
|
|
|
|
|
|
install = make_install
|
|
|
|
|
|
def execute(self):
|
|
'''
|
|
Runs appropriate steps for *building* this source code tree
|
|
'''
|
|
self.configure()
|
|
self.make()
|
|
|
|
|
|
class GnuSourceBuildParamHelper(GnuSourceBuildHelper):
|
|
'''
|
|
Helps to deal with gnu_autotools build helper in cartersian config files
|
|
|
|
This class attempts to make it simple to build source coude, by using a
|
|
naming standard that follows this basic syntax:
|
|
|
|
[<git_repo>|<local_src>]_<name>_<option> = value
|
|
|
|
To pass extra options to the configure script, while building foo from a
|
|
git repo, set the following variable:
|
|
|
|
git_repo_foo_configure_options = --enable-feature
|
|
'''
|
|
def __init__(self, params, name, destination_dir, install_prefix):
|
|
'''
|
|
Instantiates a new GnuSourceBuildParamHelper
|
|
'''
|
|
self.params = params
|
|
self.name = name
|
|
self.destination_dir = destination_dir
|
|
self.install_prefix = install_prefix
|
|
self._parse_params()
|
|
|
|
|
|
def _parse_params(self):
|
|
'''
|
|
Parses the params items for entries related to source directory
|
|
|
|
This method currently does everything that the parent class __init__()
|
|
method does, that is, sets all instance variables needed by other
|
|
methods. That means it's not strictly necessary to call parent's
|
|
__init__().
|
|
'''
|
|
logging.debug('Parsing gnu_autotools build parameters for %s' %
|
|
self.name)
|
|
|
|
configure_opt_key = '%s_configure_options' % self.name
|
|
configure_options = self.params.get(configure_opt_key, '').split()
|
|
logging.debug('Configure options for %s: %s' % (self.name,
|
|
configure_options))
|
|
|
|
self.source = self.destination_dir
|
|
self.build_dir = self.destination_dir
|
|
self.prefix = self.install_prefix
|
|
self.configure_options = configure_options
|
|
self.include_pkg_config_path()
|
|
|
|
|
|
def install_host_kernel(job, params):
|
|
"""
|
|
Install a host kernel, given the appropriate params.
|
|
|
|
@param job: Job object.
|
|
@param params: Dict with host kernel install params.
|
|
"""
|
|
install_type = params.get('host_kernel_install_type')
|
|
|
|
rpm_url = params.get('host_kernel_rpm_url')
|
|
|
|
koji_cmd = params.get('host_kernel_koji_cmd')
|
|
koji_build = params.get('host_kernel_koji_build')
|
|
koji_tag = params.get('host_kernel_koji_tag')
|
|
|
|
git_repo = params.get('host_kernel_git_repo')
|
|
git_branch = params.get('host_kernel_git_branch')
|
|
git_commit = params.get('host_kernel_git_commit')
|
|
patch_list = params.get('host_kernel_patch_list')
|
|
if patch_list:
|
|
patch_list = patch_list.split()
|
|
kernel_config = params.get('host_kernel_config')
|
|
|
|
if install_type == 'rpm':
|
|
logging.info('Installing host kernel through rpm')
|
|
dst = os.path.join("/tmp", os.path.basename(rpm_url))
|
|
k = utils.get_file(rpm_url, dst)
|
|
host_kernel = job.kernel(k)
|
|
host_kernel.install(install_vmlinux=False)
|
|
host_kernel.boot()
|
|
|
|
elif install_type in ['koji', 'brew']:
|
|
k_deps = KojiPkgSpec(tag=koji_tag, package='kernel',
|
|
subpackages=['kernel-devel', 'kernel-firmware'])
|
|
k = KojiPkgSpec(tag=koji_tag, package='kernel',
|
|
subpackages=['kernel'])
|
|
|
|
c = KojiClient(koji_cmd)
|
|
logging.info('Fetching kernel dependencies (-devel, -firmware)')
|
|
c.get_pkgs(k_deps, job.tmpdir)
|
|
logging.info('Installing kernel dependencies (-devel, -firmware) '
|
|
'through %s', install_type)
|
|
k_deps_rpm_file_names = [os.path.join(job.tmpdir, rpm_file_name) for
|
|
rpm_file_name in c.get_pkg_rpm_file_names(k_deps)]
|
|
utils.run('rpm -U --force %s' % " ".join(k_deps_rpm_file_names))
|
|
|
|
c.get_pkgs(k, job.tmpdir)
|
|
k_rpm = os.path.join(job.tmpdir,
|
|
c.get_pkg_rpm_file_names(k)[0])
|
|
host_kernel = job.kernel(k_rpm)
|
|
host_kernel.install(install_vmlinux=False)
|
|
host_kernel.boot()
|
|
|
|
elif install_type == 'git':
|
|
logging.info('Chose to install host kernel through git, proceeding')
|
|
repodir = os.path.join("/tmp", 'kernel_src')
|
|
r = get_git_branch(git_repo, git_branch, repodir, git_commit)
|
|
host_kernel = job.kernel(r)
|
|
if patch_list:
|
|
host_kernel.patch(patch_list)
|
|
host_kernel.config(kernel_config)
|
|
host_kernel.build()
|
|
host_kernel.install()
|
|
host_kernel.boot()
|
|
|
|
else:
|
|
logging.info('Chose %s, using the current kernel for the host',
|
|
install_type)
|
|
|
|
|
|
def if_nametoindex(ifname):
|
|
"""
|
|
Map an interface name into its corresponding index.
|
|
Returns 0 on error, as 0 is not a valid index
|
|
|
|
@param ifname: interface name
|
|
"""
|
|
index = 0
|
|
ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
|
|
ifr = struct.pack("16si", ifname, 0)
|
|
r = fcntl.ioctl(ctrl_sock, SIOCGIFINDEX, ifr)
|
|
index = struct.unpack("16si", r)[1]
|
|
ctrl_sock.close()
|
|
return index
|
|
|
|
|
|
def vnet_hdr_probe(tapfd):
|
|
"""
|
|
Check if the IFF_VNET_HDR is support by tun.
|
|
|
|
@param tapfd: the file descriptor of /dev/net/tun
|
|
"""
|
|
u = struct.pack("I", 0)
|
|
try:
|
|
r = fcntl.ioctl(tapfd, TUNGETFEATURES, u)
|
|
except OverflowError:
|
|
return False
|
|
flags = struct.unpack("I", r)[0]
|
|
if flags & IFF_VNET_HDR:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def open_tap(devname, ifname, vnet_hdr=True):
|
|
"""
|
|
Open a tap device and returns its file descriptor which is used by
|
|
fd=<fd> parameter of qemu-kvm.
|
|
|
|
@param ifname: TAP interface name
|
|
@param vnet_hdr: Whether enable the vnet header
|
|
"""
|
|
try:
|
|
tapfd = os.open(devname, os.O_RDWR)
|
|
except OSError, e:
|
|
raise TAPModuleError(devname, "open", e)
|
|
flags = IFF_TAP | IFF_NO_PI
|
|
if vnet_hdr and vnet_hdr_probe(tapfd):
|
|
flags |= IFF_VNET_HDR
|
|
|
|
ifr = struct.pack("16sh", ifname, flags)
|
|
try:
|
|
r = fcntl.ioctl(tapfd, TUNSETIFF, ifr)
|
|
except IOError, details:
|
|
raise TAPCreationError(ifname, details)
|
|
ifname = struct.unpack("16sh", r)[0].strip("\x00")
|
|
return tapfd
|
|
|
|
|
|
def add_to_bridge(ifname, brname):
|
|
"""
|
|
Add a TAP device to bridge
|
|
|
|
@param ifname: Name of TAP device
|
|
@param brname: Name of the bridge
|
|
"""
|
|
ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
|
|
index = if_nametoindex(ifname)
|
|
if index == 0:
|
|
raise TAPNotExistError(ifname)
|
|
ifr = struct.pack("16si", brname, index)
|
|
try:
|
|
r = fcntl.ioctl(ctrl_sock, SIOCBRADDIF, ifr)
|
|
except IOError, details:
|
|
raise BRAddIfError(ifname, brname, details)
|
|
ctrl_sock.close()
|
|
|
|
|
|
def bring_up_ifname(ifname):
|
|
"""
|
|
Bring up an interface
|
|
|
|
@param ifname: Name of the interface
|
|
"""
|
|
ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
|
|
ifr = struct.pack("16si", ifname, IFF_UP)
|
|
try:
|
|
fcntl.ioctl(ctrl_sock, SIOCSIFFLAGS, ifr)
|
|
except IOError:
|
|
raise TAPBringUpError(ifname)
|
|
ctrl_sock.close()
|
|
|
|
|
|
def if_set_macaddress(ifname, mac):
|
|
"""
|
|
Set the mac address for an interface
|
|
|
|
@param ifname: Name of the interface
|
|
@mac: Mac address
|
|
"""
|
|
ctrl_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
|
|
|
|
ifr = struct.pack("256s", ifname)
|
|
try:
|
|
mac_dev = fcntl.ioctl(ctrl_sock, SIOCGIFHWADDR, ifr)[18:24]
|
|
mac_dev = ":".join(["%02x" % ord(m) for m in mac_dev])
|
|
except IOError, e:
|
|
raise HwAddrGetError(ifname)
|
|
|
|
if mac_dev.lower() == mac.lower():
|
|
return
|
|
|
|
ifr = struct.pack("16sH14s", ifname, 1,
|
|
"".join([chr(int(m, 16)) for m in mac.split(":")]))
|
|
try:
|
|
fcntl.ioctl(ctrl_sock, SIOCSIFHWADDR, ifr)
|
|
except IOError, e:
|
|
logging.info(e)
|
|
raise HwAddrSetError(ifname, mac)
|
|
ctrl_sock.close()
|