508 lines
21 KiB
Python
508 lines
21 KiB
Python
#!/usr/bin/python
|
|
|
|
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
import logging
|
|
|
|
import common
|
|
|
|
import httplib
|
|
import httplib2
|
|
from autotest_lib.server.cros.dynamic_suite import constants
|
|
from chromite.lib import gdata_lib
|
|
|
|
try:
|
|
from apiclient.discovery import build as apiclient_build
|
|
from apiclient import errors as apiclient_errors
|
|
from oauth2client import file as oauth_client_fileio
|
|
except ImportError as e:
|
|
apiclient_build = None
|
|
logging.debug("API client for bug filing disabled. %s", e)
|
|
|
|
|
|
class ProjectHostingApiException(Exception):
|
|
"""
|
|
Raised when an api call fails, since the actual
|
|
HTTP error can be cryptic.
|
|
"""
|
|
|
|
|
|
class BaseIssue(gdata_lib.Issue):
|
|
"""Base issue class with the minimum data to describe a tracker bug.
|
|
"""
|
|
def __init__(self, t_issue):
|
|
kwargs = {}
|
|
kwargs.update((keys, t_issue.get(keys))
|
|
for keys in gdata_lib.Issue.SlotDefaults.keys())
|
|
super(BaseIssue, self).__init__(**kwargs)
|
|
|
|
|
|
class Issue(BaseIssue):
|
|
"""
|
|
Class representing an Issue and it's related metadata.
|
|
"""
|
|
def __init__(self, t_issue):
|
|
"""
|
|
Initialize |self| from tracker issue |t_issue|
|
|
|
|
@param t_issue: The base issue we want to use to populate
|
|
the member variables of this object.
|
|
|
|
@raises ProjectHostingApiException: If the tracker issue doesn't
|
|
contain all expected fields needed to create a complete issue.
|
|
"""
|
|
super(Issue, self).__init__(t_issue)
|
|
|
|
try:
|
|
# The value keyed under 'summary' in the tracker issue
|
|
# is, unfortunately, not the summary but the title. The
|
|
# actual summary is the update at index 0.
|
|
self.summary = t_issue.get('updates')[0]
|
|
self.comments = t_issue.get('updates')[1:]
|
|
|
|
# open or closed statuses are classified according to labels like
|
|
# unconfirmed, verified, fixed etc just like through the front end.
|
|
self.state = t_issue.get(constants.ISSUE_STATE)
|
|
self.merged_into = None
|
|
if (t_issue.get(constants.ISSUE_STATUS)
|
|
== constants.ISSUE_DUPLICATE and
|
|
constants.ISSUE_MERGEDINTO in t_issue):
|
|
parent_issue_dict = t_issue.get(constants.ISSUE_MERGEDINTO)
|
|
self.merged_into = parent_issue_dict.get('issueId')
|
|
except KeyError as e:
|
|
raise ProjectHostingApiException('Cannot create a '
|
|
'complete issue %s, tracker issue: %s' % (e, t_issue))
|
|
|
|
|
|
class ProjectHostingApiClient():
|
|
"""
|
|
Client class for interaction with the project hosting api.
|
|
"""
|
|
|
|
# Maximum number of results we would like when querying the tracker.
|
|
_max_results_for_issue = 50
|
|
_start_index = 1
|
|
|
|
|
|
def __init__(self, oauth_credentials, project_name,
|
|
monorail_server='staging'):
|
|
if apiclient_build is None:
|
|
raise ProjectHostingApiException('Cannot get apiclient library.')
|
|
|
|
if not oauth_credentials:
|
|
raise ProjectHostingApiException('No oauth_credentials is provided.')
|
|
|
|
# TODO(akeshet): This try-except is due to incompatibility of phapi_lib
|
|
# with oauth2client > 2. Until this is fixed, this is expected to fail
|
|
# and bug filing will be effectively disabled. crbug.com/648489
|
|
try:
|
|
storage = oauth_client_fileio.Storage(oauth_credentials)
|
|
credentials = storage.get()
|
|
except Exception as e:
|
|
raise ProjectHostingApiException('Incompaible credentials format, '
|
|
'or other exception. Will not file '
|
|
'bugs.')
|
|
if credentials is None or credentials.invalid:
|
|
raise ProjectHostingApiException('Invalid credentials for Project '
|
|
'Hosting api. Cannot file bugs.')
|
|
|
|
http = credentials.authorize(httplib2.Http())
|
|
try:
|
|
url = ('https://monorail-%s.appspot.com/_ah/api/discovery/v1/'
|
|
'apis/{api}/{apiVersion}/rest' % monorail_server)
|
|
self._codesite_service = apiclient_build(
|
|
"monorail", "v1", http=http,
|
|
discoveryServiceUrl=url)
|
|
except (apiclient_errors.Error, httplib2.HttpLib2Error,
|
|
httplib.BadStatusLine) as e:
|
|
raise ProjectHostingApiException(str(e))
|
|
self._project_name = project_name
|
|
|
|
|
|
def _execute_request(self, request):
|
|
"""
|
|
Executes an api request.
|
|
|
|
@param request: An apiclient.http.HttpRequest object representing the
|
|
request to be executed.
|
|
@raises: ProjectHostingApiException if we fail to execute the request.
|
|
This could happen if we receive an http response that is not a
|
|
2xx, or if the http object itself encounters an error.
|
|
|
|
@return: A deserialized object model of the response body returned for
|
|
the request.
|
|
"""
|
|
try:
|
|
return request.execute()
|
|
except (apiclient_errors.Error, httplib2.HttpLib2Error,
|
|
httplib.BadStatusLine) as e:
|
|
msg = 'Unable to execute your request: %s'
|
|
raise ProjectHostingApiException(msg % e)
|
|
|
|
|
|
def _get_field(self, field):
|
|
"""
|
|
Gets a field from the project.
|
|
|
|
This method directly queries the project hosting API using bugdroids1's,
|
|
api key.
|
|
|
|
@param field: A selector, which corresponds loosely to a field in the
|
|
new bug description of the crosbug frontend.
|
|
@raises: ProjectHostingApiException, if the request execution fails.
|
|
|
|
@return: A json formatted python dict of the specified field's options,
|
|
or None if we can't find the api library. This dictionary
|
|
represents the javascript literal used by the front end tracker
|
|
and can hold multiple filds.
|
|
|
|
The returned dictionary follows a template, but it's structure
|
|
is only loosely defined as it needs to match whatever the front
|
|
end describes via javascript.
|
|
For a new issue interface which looks like:
|
|
|
|
field 1: text box
|
|
drop down: predefined value 1 = description
|
|
predefined value 2 = description
|
|
field 2: text box
|
|
similar structure as field 1
|
|
|
|
you will get a dictionary like:
|
|
{
|
|
'field name 1': {
|
|
'project realted config': 'config value'
|
|
'property': [
|
|
{predefined value for property 1, description},
|
|
{predefined value for property 2, description}
|
|
]
|
|
},
|
|
|
|
'field name 2': {
|
|
similar structure
|
|
}
|
|
...
|
|
}
|
|
"""
|
|
project = self._codesite_service.projects()
|
|
request = project.get(projectId=self._project_name,
|
|
fields=field)
|
|
return self._execute_request(request)
|
|
|
|
|
|
def _list_updates(self, issue_id):
|
|
"""
|
|
Retrieve all updates for a given issue including comments, changes to
|
|
it's labels, status etc. The first element in the dictionary returned
|
|
by this method, is by default, the 0th update on the bug; which is the
|
|
entry that created it. All the text in a given update is keyed as
|
|
'content', and updates that contain no text, eg: a change to the status
|
|
of a bug, will contain the emtpy string instead.
|
|
|
|
@param issue_id: The id of the issue we want detailed information on.
|
|
@raises: ProjectHostingApiException, if the request execution fails.
|
|
|
|
@return: A json formatted python dict that has an entry for each update
|
|
performed on this issue.
|
|
"""
|
|
issue_comments = self._codesite_service.issues().comments()
|
|
request = issue_comments.list(projectId=self._project_name,
|
|
issueId=issue_id,
|
|
maxResults=self._max_results_for_issue)
|
|
return self._execute_request(request)
|
|
|
|
|
|
def _get_issue(self, issue_id):
|
|
"""
|
|
Gets an issue given it's id.
|
|
|
|
@param issue_id: A string representing the issue id.
|
|
@raises: ProjectHostingApiException, if failed to get the issue.
|
|
|
|
@return: A json formatted python dict that has the issue content.
|
|
"""
|
|
issues = self._codesite_service.issues()
|
|
try:
|
|
request = issues.get(projectId=self._project_name,
|
|
issueId=issue_id)
|
|
except TypeError as e:
|
|
raise ProjectHostingApiException(
|
|
'Unable to get issue %s from project %s: %s' %
|
|
(issue_id, self._project_name, str(e)))
|
|
return self._execute_request(request)
|
|
|
|
|
|
def set_max_results(self, max_results):
|
|
"""Set the max results to return.
|
|
|
|
@param max_results: An integer representing the maximum number of
|
|
matching results to return per query.
|
|
"""
|
|
self._max_results_for_issue = max_results
|
|
|
|
|
|
def set_start_index(self, start_index):
|
|
"""Set the start index, for paging.
|
|
|
|
@param start_index: The new start index to use.
|
|
"""
|
|
self._start_index = start_index
|
|
|
|
|
|
def list_issues(self, **kwargs):
|
|
"""
|
|
List issues containing the search marker. This method will only list
|
|
the summary, title and id of an issue, though it searches through the
|
|
comments. Eg: if we're searching for the marker '123', issues that
|
|
contain a comment of '123' will appear in the output, but the string
|
|
'123' itself may not, because the output only contains issue summaries.
|
|
|
|
@param kwargs:
|
|
q: The anchor string used in the search.
|
|
can: a string representing the search space that is passed to the
|
|
google api, can be 'all', 'new', 'open', 'owned', 'reported',
|
|
'starred', or 'to-verify', defaults to 'all'.
|
|
label: A string representing a single label to match.
|
|
|
|
@return: A json formatted python dict of all matching issues.
|
|
|
|
@raises: ProjectHostingApiException, if the request execution fails.
|
|
"""
|
|
issues = self._codesite_service.issues()
|
|
|
|
# Asking for issues with None or '' labels will restrict the query
|
|
# to those issues without labels.
|
|
if not kwargs['label']:
|
|
del kwargs['label']
|
|
|
|
request = issues.list(projectId=self._project_name,
|
|
startIndex=self._start_index,
|
|
maxResults=self._max_results_for_issue,
|
|
**kwargs)
|
|
return self._execute_request(request)
|
|
|
|
|
|
def _get_property_values(self, prop_dict):
|
|
"""
|
|
Searches a dictionary as returned by _get_field for property lists,
|
|
then returns each value in the list. Effectively this gives us
|
|
all the accepted values for a property. For example, in crosbug,
|
|
'properties' map to things like Status, Labels, Owner etc, each of these
|
|
will have a list within the issuesConfig dict.
|
|
|
|
@param prop_dict: dictionary which contains a list of properties.
|
|
@yield: each value in a property list. This can be a dict or any other
|
|
type of datastructure, the caller is responsible for handling
|
|
it correctly.
|
|
"""
|
|
for name, property in prop_dict.iteritems():
|
|
if isinstance(property, list):
|
|
for values in property:
|
|
yield values
|
|
|
|
|
|
def _get_cros_labels(self, prop_dict):
|
|
"""
|
|
Helper function to isolate labels from the labels dictionary. This
|
|
dictionary is of the form:
|
|
{
|
|
"label": "Cr-OS-foo",
|
|
"description": "description"
|
|
},
|
|
And maps to the frontend like so:
|
|
Labels: Cr-???
|
|
Cr-OS-foo = description
|
|
where Cr-OS-foo is a conveniently predefined value for Label Cr-OS-???.
|
|
|
|
@param prop_dict: a dictionary we expect the Cros label to be in.
|
|
@return: A lower case product area, eg: video, factory, ui.
|
|
"""
|
|
label = prop_dict.get('label')
|
|
if label and 'Cr-OS-' in label:
|
|
return label.split('Cr-OS-')[1]
|
|
|
|
|
|
def get_areas(self):
|
|
"""
|
|
Parse issue options and return a list of 'Cr-OS' labels.
|
|
|
|
@return: a list of Cr-OS labels from crosbug, eg: ['kernel', 'systems']
|
|
"""
|
|
if apiclient_build is None:
|
|
logging.error('Missing Api-client import. Cannot get area-labels.')
|
|
return []
|
|
|
|
try:
|
|
issue_options_dict = self._get_field('issuesConfig')
|
|
except ProjectHostingApiException as e:
|
|
logging.error('Unable to determine area labels: %s', str(e))
|
|
return []
|
|
|
|
# Since we can request multiple fields at once we need to
|
|
# retrieve each one from the field options dictionary, even if we're
|
|
# really only asking for one field.
|
|
issue_options = issue_options_dict.get('issuesConfig')
|
|
if issue_options is None:
|
|
logging.error('The IssueConfig field does not contain issue '
|
|
'configuration as a member anymore; The project '
|
|
'hosting api might have changed.')
|
|
return []
|
|
|
|
return filter(None, [self._get_cros_labels(each)
|
|
for each in self._get_property_values(issue_options)
|
|
if isinstance(each, dict)])
|
|
|
|
|
|
def create_issue(self, request_body):
|
|
"""
|
|
Convert the request body into an issue on the frontend tracker.
|
|
|
|
@param request_body: A python dictionary with key-value pairs
|
|
that represent the fields of the issue.
|
|
eg: {
|
|
'title': 'bug title',
|
|
'description': 'bug description',
|
|
'labels': ['Type-Bug'],
|
|
'owner': {'name': 'owner@'},
|
|
'cc': [{'name': 'cc1'}, {'name': 'cc2'}],
|
|
'components': ["Internals->Components"]
|
|
}
|
|
Note the title and descriptions fields of a
|
|
new bug are not optional, all other fields are.
|
|
@raises: ProjectHostingApiException, if request execution fails.
|
|
|
|
@return: The response body, which will contain the metadata of the
|
|
issue created, or an error response code and information
|
|
about a failure.
|
|
"""
|
|
issues = self._codesite_service.issues()
|
|
request = issues.insert(projectId=self._project_name, sendEmail=True,
|
|
body=request_body)
|
|
return self._execute_request(request)
|
|
|
|
|
|
def update_issue(self, issue_id, request_body):
|
|
"""
|
|
Convert the request body into an update on an issue.
|
|
|
|
@param request_body: A python dictionary with key-value pairs
|
|
that represent the fields of the update.
|
|
eg:
|
|
{
|
|
'content': 'comment to add',
|
|
'updates':
|
|
{
|
|
'labels': ['Type-Bug', 'another label'],
|
|
'owner': 'owner@',
|
|
'cc': ['cc1@', cc2@'],
|
|
}
|
|
}
|
|
Note the owner and cc fields need to be email
|
|
addresses the tracker recognizes.
|
|
@param issue_id: The id of the issue to update.
|
|
@raises: ProjectHostingApiException, if request execution fails.
|
|
|
|
@return: The response body, which will contain information about the
|
|
update of said issue, or an error response code and information
|
|
about a failure.
|
|
"""
|
|
issues = self._codesite_service.issues()
|
|
request = issues.comments().insert(projectId=self._project_name,
|
|
issueId=issue_id, sendEmail=False,
|
|
body=request_body)
|
|
return self._execute_request(request)
|
|
|
|
|
|
def _populate_issue_updates(self, t_issue):
|
|
"""
|
|
Populates a tracker issue with updates.
|
|
|
|
Any issue is useless without it's updates, since the updates will
|
|
contain both the summary and the comments. We need at least one of
|
|
those to successfully dedupe. The Api doesn't allow us to grab all this
|
|
information in one shot because viewing the comments on an issue
|
|
requires more authority than just viewing it's title.
|
|
|
|
@param t_issue: The basic tracker issue, to populate with updates.
|
|
@raises: ProjectHostingApiException, if request execution fails.
|
|
|
|
@returns: A tracker issue, with it's updates.
|
|
"""
|
|
updates = self._list_updates(t_issue['id'])
|
|
t_issue['updates'] = [update['content'] for update in
|
|
self._get_property_values(updates)
|
|
if update.get('content')]
|
|
return t_issue
|
|
|
|
|
|
def get_tracker_issues_by_text(self, search_text, full_text=True,
|
|
include_dupes=False, label=None):
|
|
"""
|
|
Find all Tracker issues that contain the specified search text.
|
|
|
|
@param search_text: Anchor text to use in the search.
|
|
@param full_text: True if we would like an extensive search through
|
|
issue comments. If False the search will be restricted
|
|
to just summaries and titles.
|
|
@param include_dupes: If True, search over both open issues as well as
|
|
closed issues whose status is 'Duplicate'. If False,
|
|
only search over open issues.
|
|
@param label: A string representing a single label to match.
|
|
|
|
@return: A list of issues that contain the search text, or an empty list
|
|
when we're either unable to list issues or none match the text.
|
|
"""
|
|
issue_list = []
|
|
try:
|
|
search_space = 'all' if include_dupes else 'open'
|
|
feed = self.list_issues(q=search_text, can=search_space,
|
|
label=label)
|
|
except ProjectHostingApiException as e:
|
|
logging.error('Unable to search for issues with marker %s: %s',
|
|
search_text, e)
|
|
return issue_list
|
|
|
|
for t_issue in self._get_property_values(feed):
|
|
state = t_issue.get(constants.ISSUE_STATE)
|
|
status = t_issue.get(constants.ISSUE_STATUS)
|
|
is_open_or_dup = (state == constants.ISSUE_OPEN or
|
|
(state == constants.ISSUE_CLOSED
|
|
and status == constants.ISSUE_DUPLICATE))
|
|
# All valid issues will have an issue id we can use to retrieve
|
|
# more information about it. If we encounter a failure mode that
|
|
# returns a bad Http response code but doesn't throw an exception
|
|
# we won't find an issue id in the returned json.
|
|
if t_issue.get('id') and is_open_or_dup:
|
|
# TODO(beeps): If this method turns into a performance
|
|
# bottle neck yield each issue and refactor the reporter.
|
|
# For now passing all issues allows us to detect when
|
|
# deduping fails, because multiple issues will match a
|
|
# given query exactly.
|
|
try:
|
|
if full_text:
|
|
issue = Issue(self._populate_issue_updates(t_issue))
|
|
else:
|
|
issue = BaseIssue(t_issue)
|
|
except ProjectHostingApiException as e:
|
|
logging.error('Unable to list the updates of issue %s: %s',
|
|
t_issue.get('id'), str(e))
|
|
else:
|
|
issue_list.append(issue)
|
|
return issue_list
|
|
|
|
|
|
def get_tracker_issue_by_id(self, issue_id):
|
|
"""
|
|
Returns an issue object given the id.
|
|
|
|
@param issue_id: A string representing the issue id.
|
|
|
|
@return: An Issue object on success or None on failure.
|
|
"""
|
|
try:
|
|
t_issue = self._get_issue(issue_id)
|
|
return Issue(self._populate_issue_updates(t_issue))
|
|
except ProjectHostingApiException as e:
|
|
logging.error('Creation of an Issue object for %s fails: %s',
|
|
issue_id, str(e))
|
|
return None
|