5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 2 only,
9 # as published by the Free Software Foundation.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License version 2 for more details (a copy is included
15 # in the LICENSE file that accompanied this code).
17 # You should have received a copy of the GNU General Public License
18 # version 2 along with this program; If not, see
19 # http://www.gnu.org/licenses/gpl-2.0.html
23 # Copyright (c) 2014, Intel Corporation.
25 # Author: John L. Hammond <john.hammond@intel.com>
28 Gerrit Checkpatch Reviewer Daemon
29 ~~~~~~ ~~~~~~~~~~ ~~~~~~~~ ~~~~~~
31 * Watch for new change revisions in a gerrit instance.
32 * Pass new revisions through checkpatch script.
33 * POST reviews back to gerrit based on checkpatch output.
46 def _getenv_list(key, default=None, sep=':'):
48 'PATH' => ['/bin', '/usr/bin', ...]
50 value = os.getenv(key)
54 return value.split(sep)
56 GERRIT_HOST = os.getenv('GERRIT_HOST', 'review.whamcloud.com')
57 GERRIT_PROJECT = os.getenv('GERRIT_PROJECT', 'fs/lustre-release')
58 GERRIT_BRANCH = os.getenv('GERRIT_BRANCH', 'master')
59 GERRIT_AUTH_PATH = os.getenv('GERRIT_AUTH_PATH', 'GERRIT_AUTH')
61 # GERRIT_AUTH should contain a single JSON dictionary of the form:
63 # "review.example.com": {
65 # "username": "example-checkpatch",
72 CHECKPATCH_PATHS = _getenv_list('CHECKPATCH_PATHS', ['checkpatch.pl'])
73 CHECKPATCH_IGNORED_FILES = _getenv_list('CHECKPATCH_IGNORED_FILES', [
74 'lustre/contrib/wireshark/packet-lustre.c',
75 'lustre/ptlrpc/wiretest.c',
76 'lustre/utils/wiretest.c',
78 CHECKPATCH_IGNORED_KINDS = _getenv_list('CHECKPATCH_IGNORED_KINDS', [
80 REVIEW_HISTORY_PATH = os.getenv('REVIEW_HISTORY_PATH', 'REVIEW_HISTORY')
81 STYLE_LINK = os.getenv('STYLE_LINK',
82 'https://wiki.hpdd.intel.com/display/PUB/Coding+Guidelines')
84 USE_CODE_REVIEW_SCORE = False
86 def parse_checkpatch_output(out, path_line_comments, warning_count):
88 Parse string output out of CHECKPATCH into path_line_comments.
89 Increment warning_count[0] for each warning.
91 path_line_comments is { PATH: { LINE: [COMMENT, ...] }, ... }.
93 def add_comment(path, line, level, kind, message):
95 logging.debug("add_comment %s %d %s %s '%s'",
96 path, line, level, kind, message)
97 if kind in CHECKPATCH_IGNORED_KINDS:
100 for pattern in CHECKPATCH_IGNORED_FILES:
101 if fnmatch.fnmatch(path, pattern):
104 path_comments = path_line_comments.setdefault(path, {})
105 line_comments = path_comments.setdefault(line, [])
106 line_comments.append('(style) ' + message)
107 warning_count[0] += 1
109 level = None # 'ERROR', 'WARNING'
110 kind = None # 'CODE_INDENT', 'LEADING_SPACE', ...
111 message = None # 'code indent should use tabs where possible'
113 for line in out.splitlines():
114 # ERROR:CODE_INDENT: code indent should use tabs where possible
115 # #404: FILE: lustre/liblustre/dir.c:103:
116 # + op_data.op_hash_offset = hash_x_index(page->index, 0);$
119 level, kind, message = None, None, None
121 # '#404: FILE: lustre/liblustre/dir.c:103:'
122 tokens = line.split(':', 5)
123 if len(tokens) != 5 or tokens[1] != ' FILE':
126 path = tokens[2].strip()
127 line_number_str = tokens[3].strip()
128 if not line_number_str.isdigit():
131 line_number = int(line_number_str)
133 if path and level and kind and message:
134 add_comment(path, line_number, level, kind, message)
138 # ERROR:CODE_INDENT: code indent should use tabs where possible
140 level, kind, message = line.split(':', 2)
142 level, kind, message = None, None, None
144 if level != 'ERROR' and level != 'WARNING':
145 level, kind, message = None, None, None
148 def review_input_and_score(path_line_comments, warning_count):
150 Convert { PATH: { LINE: [COMMENT, ...] }, ... }, [11] to a gerrit
151 ReviewInput() and score
155 for path, line_comments in path_line_comments.iteritems():
157 for line, comment_list in line_comments.iteritems():
158 message = '\n'.join(comment_list)
159 path_comments.append({'line': line, 'message': message})
160 review_comments[path] = path_comments
162 if warning_count[0] > 0:
167 if USE_CODE_REVIEW_SCORE:
168 code_review_score = score
170 code_review_score = 0
174 'message': ('%d style warning(s).\nFor more details please see %s' %
175 (warning_count[0], STYLE_LINK)),
177 'Code-Review': code_review_score
179 'comments': review_comments
183 'message': 'Looks good to me.',
185 'Code-Review': code_review_score
192 return long(time.time())
195 class Reviewer(object):
197 * Poll gerrit instance for updates to changes matching project and branch.
198 * Pipe new patches through checkpatch.
199 * Convert checkpatch output to gerrit ReviewInput().
200 * Post ReviewInput() to gerrit instance.
201 * Track reviewed revisions in history_path.
203 def __init__(self, host, project, branch, username, password, history_path):
205 self.project = project
207 self.auth = requests.auth.HTTPDigestAuth(username, password)
208 self.logger = logging.getLogger(__name__)
209 self.history_path = history_path
210 self.history_mode = 'rw'
213 self.post_enabled = True
214 self.post_interval = 10
215 self.update_interval = 300
217 def _debug(self, msg, *args):
219 self.logger.debug(msg, *args)
221 def _error(self, msg, *args):
223 self.logger.error(msg, *args)
225 def _url(self, path):
227 return 'http://' + self.host + '/a' + path
229 def _get(self, path):
231 GET path return Response.
233 url = self._url(path)
235 res = requests.get(url, auth=self.auth)
236 except requests.exceptions.RequestException as exc:
237 self._error("cannot GET '%s': exception = %s", url, str(exc))
240 if res.status_code != requests.codes.ok:
241 self._error("cannot GET '%s': reason = %s, status_code = %d",
242 url, res.reason, res.status_code)
247 def _post(self, path, obj):
249 POST json(obj) to path, return True on success.
251 url = self._url(path)
252 data = json.dumps(obj)
253 if not self.post_enabled:
254 self._debug("_post: disabled: url = '%s', data = '%s'", url, data)
258 res = requests.post(url, data=data,
259 headers={'Content-Type': 'application/json'},
261 except requests.exceptions.RequestException as exc:
262 self._error("cannot POST '%s': exception = %s", url, str(exc))
265 if res.status_code != requests.codes.ok:
266 self._error("cannot POST '%s': reason = %s, status_code = %d",
267 url, res.reason, res.status_code)
272 def load_history(self):
274 Load review history from history_path containing lines of the form:
275 EPOCH FULL_CHANGE_ID REVISION SCORE
276 1394536722 fs%2Flustre-release~master~I5cc6c23... 00e2cc75... 1
278 1394537033 fs%2Flustre-release~master~I10be8e9... 44f7b504... 1
283 if 'r' in self.history_mode:
284 with open(self.history_path) as history_file:
285 for line in history_file:
286 epoch, change_id, revision, score = line.split()
288 self.timestamp = long(float(epoch))
290 self.history[change_id + ' ' + revision] = score
292 self._debug("load_history: history size = %d, timestamp = %d",
293 len(self.history), self.timestamp)
295 def write_history(self, change_id, revision, score, epoch=-1):
297 Add review record to history dict and file.
300 self.history[change_id + ' ' + revision] = score
303 epoch = self.timestamp
305 if 'w' in self.history_mode:
306 with open(self.history_path, 'a') as history_file:
307 print >> history_file, epoch, change_id, revision, score
309 def in_history(self, change_id, revision):
311 Return True if change_id/revision was already reviewed.
313 return change_id + ' ' + revision in self.history
315 def get_changes(self, query):
317 GET a list of ChangeInfo()s for all changes matching query.
319 {'status':'open', '-age':'60m'} =>
320 GET /changes/?q=project:...+status:open+-age:60m&o=CURRENT_REVISION =>
324 project = query.get('project', self.project)
325 query['project'] = urllib.quote(project, safe='')
326 branch = query.get('branch', self.branch)
327 query['branch'] = urllib.quote(branch, safe='')
328 path = ('/changes/?q=' +
329 '+'.join(k + ':' + v for k, v in query.iteritems()) +
330 '&o=CURRENT_REVISION')
331 res = self._get(path)
335 # Gerrit uses " )]}'" to guard against XSSI.
336 return json.loads(res.content[5:])
338 def decode_patch(self, content):
340 Decode gerrit's idea of base64.
342 The base64 encoded patch returned by gerrit isn't always
343 padded correctly according to b64decode. Don't know why. Work
344 around this by appending more '=' characters or truncating the
345 content until it decodes. But do try the unmodified content
348 for i in (0, 1, 2, 3, -1, -2, -3):
350 padded_content = content + (i * '=')
352 padded_content = content[:i]
355 return base64.b64decode(padded_content)
356 except TypeError as exc:
357 self._debug("decode_patch: len = %d, exception = %s",
358 len(padded_content), str(exc))
362 def get_patch(self, change, revision='current'):
364 GET and decode the (current) patch for change.
366 path = '/changes/' + change['id'] + '/revisions/' + revision + '/patch'
367 self._debug("get_patch: path = '%s'", path)
368 res = self._get(path)
372 self._debug("get_patch: len(content) = %d, content = '%s...'",
373 len(res.content), res.content[:20])
375 return self.decode_patch(res.content)
377 def set_review(self, change, revision, review_input):
379 POST review_input for the given revision of change.
381 path = '/changes/' + change['id'] + '/revisions/' + revision + '/review'
382 self._debug("set_review: path = '%s'", path)
383 return self._post(path, review_input)
385 def check_patch(self, patch):
387 Run each script in CHECKPATCH_PATHS on patch, return a
388 ReviewInput() and score.
390 path_line_comments = {}
393 for path in CHECKPATCH_PATHS:
394 pipe = subprocess.Popen([path, '--show-types', '-'],
395 stdin=subprocess.PIPE,
396 stdout=subprocess.PIPE,
397 stderr=subprocess.PIPE)
398 out, err = pipe.communicate(patch)
399 self._debug("check_patch: path = %s, out = '%s...', err = '%s...'",
400 path, out[:80], err[:80])
401 parse_checkpatch_output(out, path_line_comments, warning_count)
403 return review_input_and_score(path_line_comments, warning_count)
405 def review_change(self, change, force=False):
407 Review the current revision of change.
408 * Bail if the change isn't open (status is not 'NEW').
409 * GET the current revision from gerrit.
410 * Bail if we've already reviewed it (unless force is True).
411 * Pipe the patch through checkpatch(es).
412 * Save results to review history.
413 * POST review to gerrit.
415 self._debug("review_change: change = %s, subject = '%s'",
416 change['id'], change.get('subject', ''))
418 status = change.get('status')
420 self._debug("review_change: status = %s", status)
423 current_revision = change.get('current_revision')
424 self._debug("review_change: current_revision = '%s'", current_revision)
425 if not current_revision:
428 # Have we already checked this revision?
429 if self.in_history(change['id'], current_revision) and not force:
430 self._debug("review_change: already reviewed")
433 patch = self.get_patch(change, current_revision)
435 self._debug("review_change: no patch")
438 review_input, score = self.check_patch(patch)
439 self._debug("review_change: score = %d", score)
440 self.write_history(change['id'], current_revision, score)
441 self.set_review(change, current_revision, review_input)
442 # Don't POST more than every post_interval seconds.
443 time.sleep(self.post_interval)
447 GET recently updated changes and review as needed.
449 new_timestamp = _now()
450 age = new_timestamp - self.timestamp + 60 * 60 # 1h padding
451 self._debug("update: age = %d", age)
453 open_changes = self.get_changes({'status':'open',
454 '-age':str(age) + 's'})
455 self._debug("update: got %d open_changes", len(open_changes))
457 for change in open_changes:
458 self.review_change(change)
460 self.timestamp = new_timestamp
461 self.write_history('-', '-', 0)
465 * Load review history.
466 * Call update() every poll_interval seconds.
469 if self.timestamp <= 0:
474 time.sleep(self.update_interval)
479 logging.basicConfig(level=logging.DEBUG)
481 with open(GERRIT_AUTH_PATH) as auth_file:
482 auth = json.load(auth_file)
483 username = auth[GERRIT_HOST]['gerrit/http']['username']
484 password = auth[GERRIT_HOST]['gerrit/http']['password']
486 reviewer = Reviewer(GERRIT_HOST, GERRIT_PROJECT, GERRIT_BRANCH,
487 username, password, REVIEW_HISTORY_PATH)
491 if __name__ == "__main__":