5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 2 only,
9 # as published by the Free Software Foundation.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License version 2 for more details (a copy is included
15 # in the LICENSE file that accompanied this code).
17 # You should have received a copy of the GNU General Public License
18 # version 2 along with this program; If not, see
19 # http://www.gnu.org/licenses/gpl-2.0.html
23 # Copyright (c) 2014, Intel Corporation.
25 # Author: John L. Hammond <john.hammond@intel.com>
28 Gerrit Checkpatch Reviewer Daemon
29 ~~~~~~ ~~~~~~~~~~ ~~~~~~~~ ~~~~~~
31 * Watch for new change revisions in a gerrit instance.
32 * Pass new revisions through checkpatch script.
33 * POST reviews back to gerrit based on checkpatch output.
46 def _getenv_list(key, default=None, sep=':'):
48 'PATH' => ['/bin', '/usr/bin', ...]
50 value = os.getenv(key)
54 return value.split(sep)
56 GERRIT_HOST = os.getenv('GERRIT_HOST', 'review.whamcloud.com')
57 GERRIT_PROJECT = os.getenv('GERRIT_PROJECT', 'fs/lustre-release')
58 GERRIT_BRANCH = os.getenv('GERRIT_BRANCH', 'master')
59 GERRIT_AUTH_PATH = os.getenv('GERRIT_AUTH_PATH', 'GERRIT_AUTH')
61 # GERRIT_AUTH should contain a single JSON dictionary of the form:
63 # "review.example.com": {
65 # "username": "example-checkpatch",
72 CHECKPATCH_PATHS = _getenv_list('CHECKPATCH_PATHS', ['checkpatch.pl'])
73 CHECKPATCH_IGNORED_FILES = _getenv_list('CHECKPATCH_IGNORED_FILES', [
74 'lustre/contrib/wireshark/packet-lustre.c',
75 'lustre/ptlrpc/wiretest.c',
76 'lustre/utils/wiretest.c',
78 CHECKPATCH_IGNORED_KINDS = _getenv_list('CHECKPATCH_IGNORED_KINDS', [
80 REVIEW_HISTORY_PATH = os.getenv('REVIEW_HISTORY_PATH', 'REVIEW_HISTORY')
81 STYLE_LINK = os.getenv('STYLE_LINK',
82 'https://wiki.hpdd.intel.com/display/PUB/Coding+Guidelines')
84 USE_CODE_REVIEW_SCORE = False
86 def parse_checkpatch_output(out, path_line_comments, warning_count):
88 Parse string output out of CHECKPATCH into path_line_comments.
89 Increment warning_count[0] for each warning.
91 path_line_comments is { PATH: { LINE: [COMMENT, ...] }, ... }.
93 def add_comment(path, line, level, kind, message):
95 logging.debug("add_comment %s %d %s %s '%s'",
96 path, line, level, kind, message)
97 if kind in CHECKPATCH_IGNORED_KINDS:
100 for pattern in CHECKPATCH_IGNORED_FILES:
101 if fnmatch.fnmatch(path, pattern):
104 path_comments = path_line_comments.setdefault(path, {})
105 line_comments = path_comments.setdefault(line, [])
106 line_comments.append('(style) ' + message)
107 warning_count[0] += 1
109 level = None # 'ERROR', 'WARNING'
110 kind = None # 'CODE_INDENT', 'LEADING_SPACE', ...
111 message = None # 'code indent should use tabs where possible'
113 for line in out.splitlines():
114 # ERROR:CODE_INDENT: code indent should use tabs where possible
115 # #404: FILE: lustre/liblustre/dir.c:103:
116 # + op_data.op_hash_offset = hash_x_index(page->index, 0);$
119 level, kind, message = None, None, None
121 # '#404: FILE: lustre/liblustre/dir.c:103:'
122 tokens = line.split(':', 5)
123 if len(tokens) != 5 or tokens[1] != ' FILE':
126 path = tokens[2].strip()
127 line_number_str = tokens[3].strip()
128 if not line_number_str.isdigit():
131 line_number = int(line_number_str)
133 if path and level and kind and message:
134 add_comment(path, line_number, level, kind, message)
138 # ERROR:CODE_INDENT: code indent should use tabs where possible
140 level, kind, message = line.split(':', 2)
142 level, kind, message = None, None, None
144 if level != 'ERROR' and level != 'WARNING':
145 level, kind, message = None, None, None
148 def review_input_and_score(path_line_comments, warning_count):
150 Convert { PATH: { LINE: [COMMENT, ...] }, ... }, [11] to a gerrit
151 ReviewInput() and score
155 for path, line_comments in path_line_comments.iteritems():
157 for line, comment_list in line_comments.iteritems():
158 message = '\n'.join(comment_list)
159 path_comments.append({'line': line, 'message': message})
160 review_comments[path] = path_comments
162 if warning_count[0] > 0:
167 if USE_CODE_REVIEW_SCORE:
168 code_review_score = score
170 code_review_score = 0
174 'message': ('%d style warning(s).\nFor more details please see %s' %
175 (warning_count[0], STYLE_LINK)),
177 'Code-Review': code_review_score
179 'comments': review_comments,
184 'message': 'Looks good to me.',
186 'Code-Review': code_review_score
194 return long(time.time())
197 class Reviewer(object):
199 * Poll gerrit instance for updates to changes matching project and branch.
200 * Pipe new patches through checkpatch.
201 * Convert checkpatch output to gerrit ReviewInput().
202 * Post ReviewInput() to gerrit instance.
203 * Track reviewed revisions in history_path.
205 def __init__(self, host, project, branch, username, password, history_path):
207 self.project = project
209 self.auth = requests.auth.HTTPDigestAuth(username, password)
210 self.logger = logging.getLogger(__name__)
211 self.history_path = history_path
212 self.history_mode = 'rw'
215 self.post_enabled = True
216 self.post_interval = 10
217 self.update_interval = 300
219 def _debug(self, msg, *args):
221 self.logger.debug(msg, *args)
223 def _error(self, msg, *args):
225 self.logger.error(msg, *args)
227 def _url(self, path):
229 return 'http://' + self.host + '/a' + path
231 def _get(self, path):
233 GET path return Response.
235 url = self._url(path)
237 res = requests.get(url, auth=self.auth)
238 except requests.exceptions.RequestException as exc:
239 self._error("cannot GET '%s': exception = %s", url, str(exc))
242 if res.status_code != requests.codes.ok:
243 self._error("cannot GET '%s': reason = %s, status_code = %d",
244 url, res.reason, res.status_code)
249 def _post(self, path, obj):
251 POST json(obj) to path, return True on success.
253 url = self._url(path)
254 data = json.dumps(obj)
255 if not self.post_enabled:
256 self._debug("_post: disabled: url = '%s', data = '%s'", url, data)
260 res = requests.post(url, data=data,
261 headers={'Content-Type': 'application/json'},
263 except requests.exceptions.RequestException as exc:
264 self._error("cannot POST '%s': exception = %s", url, str(exc))
267 if res.status_code != requests.codes.ok:
268 self._error("cannot POST '%s': reason = %s, status_code = %d",
269 url, res.reason, res.status_code)
274 def load_history(self):
276 Load review history from history_path containing lines of the form:
277 EPOCH FULL_CHANGE_ID REVISION SCORE
278 1394536722 fs%2Flustre-release~master~I5cc6c23... 00e2cc75... 1
280 1394537033 fs%2Flustre-release~master~I10be8e9... 44f7b504... 1
285 if 'r' in self.history_mode:
286 with open(self.history_path) as history_file:
287 for line in history_file:
288 epoch, change_id, revision, score = line.split()
290 self.timestamp = long(float(epoch))
292 self.history[change_id + ' ' + revision] = score
294 self._debug("load_history: history size = %d, timestamp = %d",
295 len(self.history), self.timestamp)
297 def write_history(self, change_id, revision, score, epoch=-1):
299 Add review record to history dict and file.
302 self.history[change_id + ' ' + revision] = score
305 epoch = self.timestamp
307 if 'w' in self.history_mode:
308 with open(self.history_path, 'a') as history_file:
309 print >> history_file, epoch, change_id, revision, score
311 def in_history(self, change_id, revision):
313 Return True if change_id/revision was already reviewed.
315 return change_id + ' ' + revision in self.history
317 def get_changes(self, query):
319 GET a list of ChangeInfo()s for all changes matching query.
321 {'status':'open', '-age':'60m'} =>
322 GET /changes/?q=project:...+status:open+-age:60m&o=CURRENT_REVISION =>
326 project = query.get('project', self.project)
327 query['project'] = urllib.quote(project, safe='')
328 branch = query.get('branch', self.branch)
329 query['branch'] = urllib.quote(branch, safe='')
330 path = ('/changes/?q=' +
331 '+'.join(k + ':' + v for k, v in query.iteritems()) +
332 '&o=CURRENT_REVISION')
333 res = self._get(path)
337 # Gerrit uses " )]}'" to guard against XSSI.
338 return json.loads(res.content[5:])
340 def decode_patch(self, content):
342 Decode gerrit's idea of base64.
344 The base64 encoded patch returned by gerrit isn't always
345 padded correctly according to b64decode. Don't know why. Work
346 around this by appending more '=' characters or truncating the
347 content until it decodes. But do try the unmodified content
350 for i in (0, 1, 2, 3, -1, -2, -3):
352 padded_content = content + (i * '=')
354 padded_content = content[:i]
357 return base64.b64decode(padded_content)
358 except TypeError as exc:
359 self._debug("decode_patch: len = %d, exception = %s",
360 len(padded_content), str(exc))
364 def get_patch(self, change, revision='current'):
366 GET and decode the (current) patch for change.
368 path = '/changes/' + change['id'] + '/revisions/' + revision + '/patch'
369 self._debug("get_patch: path = '%s'", path)
370 res = self._get(path)
374 self._debug("get_patch: len(content) = %d, content = '%s...'",
375 len(res.content), res.content[:20])
377 return self.decode_patch(res.content)
379 def set_review(self, change, revision, review_input):
381 POST review_input for the given revision of change.
383 path = '/changes/' + change['id'] + '/revisions/' + revision + '/review'
384 self._debug("set_review: path = '%s'", path)
385 return self._post(path, review_input)
387 def check_patch(self, patch):
389 Run each script in CHECKPATCH_PATHS on patch, return a
390 ReviewInput() and score.
392 path_line_comments = {}
395 for path in CHECKPATCH_PATHS:
396 pipe = subprocess.Popen([path, '--show-types', '-'],
397 stdin=subprocess.PIPE,
398 stdout=subprocess.PIPE,
399 stderr=subprocess.PIPE)
400 out, err = pipe.communicate(patch)
401 self._debug("check_patch: path = %s, out = '%s...', err = '%s...'",
402 path, out[:80], err[:80])
403 parse_checkpatch_output(out, path_line_comments, warning_count)
405 return review_input_and_score(path_line_comments, warning_count)
407 def review_change(self, change, force=False):
409 Review the current revision of change.
410 * Bail if the change isn't open (status is not 'NEW').
411 * GET the current revision from gerrit.
412 * Bail if we've already reviewed it (unless force is True).
413 * Pipe the patch through checkpatch(es).
414 * Save results to review history.
415 * POST review to gerrit.
417 self._debug("review_change: change = %s, subject = '%s'",
418 change['id'], change.get('subject', ''))
420 status = change.get('status')
422 self._debug("review_change: status = %s", status)
425 current_revision = change.get('current_revision')
426 self._debug("review_change: current_revision = '%s'", current_revision)
427 if not current_revision:
430 # Have we already checked this revision?
431 if self.in_history(change['id'], current_revision) and not force:
432 self._debug("review_change: already reviewed")
435 patch = self.get_patch(change, current_revision)
437 self._debug("review_change: no patch")
440 review_input, score = self.check_patch(patch)
441 self._debug("review_change: score = %d", score)
442 self.write_history(change['id'], current_revision, score)
443 self.set_review(change, current_revision, review_input)
444 # Don't POST more than every post_interval seconds.
445 time.sleep(self.post_interval)
449 GET recently updated changes and review as needed.
451 new_timestamp = _now()
452 age = new_timestamp - self.timestamp + 60 * 60 # 1h padding
453 self._debug("update: age = %d", age)
455 open_changes = self.get_changes({'status':'open',
456 '-age':str(age) + 's'})
457 self._debug("update: got %d open_changes", len(open_changes))
459 for change in open_changes:
460 self.review_change(change)
462 self.timestamp = new_timestamp
463 self.write_history('-', '-', 0)
467 * Load review history.
468 * Call update() every poll_interval seconds.
471 if self.timestamp <= 0:
476 time.sleep(self.update_interval)
481 logging.basicConfig(level=logging.DEBUG)
483 with open(GERRIT_AUTH_PATH) as auth_file:
484 auth = json.load(auth_file)
485 username = auth[GERRIT_HOST]['gerrit/http']['username']
486 password = auth[GERRIT_HOST]['gerrit/http']['password']
488 reviewer = Reviewer(GERRIT_HOST, GERRIT_PROJECT, GERRIT_BRANCH,
489 username, password, REVIEW_HISTORY_PATH)
493 if __name__ == "__main__":