5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 2 only,
9 # as published by the Free Software Foundation.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License version 2 for more details (a copy is included
15 # in the LICENSE file that accompanied this code).
17 # You should have received a copy of the GNU General Public License
18 # version 2 along with this program; If not, see
19 # http://www.gnu.org/licenses/gpl-2.0.html
23 # Copyright (c) 2014, Intel Corporation.
25 # Author: John L. Hammond <john.hammond@intel.com>
28 Gerrit Checkpatch Reviewer Daemon
29 ~~~~~~ ~~~~~~~~~~ ~~~~~~~~ ~~~~~~
31 * Watch for new change revisions in a gerrit instance.
32 * Pass new revisions through checkpatch script.
33 * POST reviews back to gerrit based on checkpatch output.
46 def _getenv_list(key, default=None, sep=':'):
48 'PATH' => ['/bin', '/usr/bin', ...]
50 value = os.getenv(key)
54 return value.split(sep)
56 GERRIT_HOST = os.getenv('GERRIT_HOST', 'review.whamcloud.com')
57 GERRIT_PROJECT = os.getenv('GERRIT_PROJECT', 'fs/lustre-release')
58 GERRIT_BRANCH = os.getenv('GERRIT_BRANCH', 'master')
59 GERRIT_AUTH_PATH = os.getenv('GERRIT_AUTH_PATH', 'GERRIT_AUTH')
61 # GERRIT_AUTH should contain a single JSON dictionary of the form:
63 # "review.example.com": {
65 # "username": "example-checkpatch",
72 CHECKPATCH_PATHS = _getenv_list('CHECKPATCH_PATHS', ['checkpatch.pl'])
73 CHECKPATCH_IGNORED_FILES = _getenv_list('CHECKPATCH_IGNORED_FILES', [
74 'lustre/contrib/wireshark/packet-lustre.c',
75 'lustre/ptlrpc/wiretest.c',
76 'lustre/utils/wiretest.c',
78 CHECKPATCH_IGNORED_KINDS = _getenv_list('CHECKPATCH_IGNORED_KINDS', [
82 REVIEW_HISTORY_PATH = os.getenv('REVIEW_HISTORY_PATH', 'REVIEW_HISTORY')
83 STYLE_LINK = os.getenv('STYLE_LINK',
84 'https://wiki.hpdd.intel.com/display/PUB/Coding+Guidelines')
86 USE_CODE_REVIEW_SCORE = False
88 def parse_checkpatch_output(out, path_line_comments, warning_count):
90 Parse string output out of CHECKPATCH into path_line_comments.
91 Increment warning_count[0] for each warning.
93 path_line_comments is { PATH: { LINE: [COMMENT, ...] }, ... }.
95 def add_comment(path, line, level, kind, message):
97 logging.debug("add_comment %s %d %s %s '%s'",
98 path, line, level, kind, message)
99 if kind in CHECKPATCH_IGNORED_KINDS:
102 for pattern in CHECKPATCH_IGNORED_FILES:
103 if fnmatch.fnmatch(path, pattern):
106 path_comments = path_line_comments.setdefault(path, {})
107 line_comments = path_comments.setdefault(line, [])
108 line_comments.append('(style) ' + message)
109 warning_count[0] += 1
111 level = None # 'ERROR', 'WARNING'
112 kind = None # 'CODE_INDENT', 'LEADING_SPACE', ...
113 message = None # 'code indent should use tabs where possible'
115 for line in out.splitlines():
116 # ERROR:CODE_INDENT: code indent should use tabs where possible
117 # #404: FILE: lustre/liblustre/dir.c:103:
118 # + op_data.op_hash_offset = hash_x_index(page->index, 0);$
121 level, kind, message = None, None, None
123 # '#404: FILE: lustre/liblustre/dir.c:103:'
124 tokens = line.split(':', 5)
125 if len(tokens) != 5 or tokens[1] != ' FILE':
128 path = tokens[2].strip()
129 line_number_str = tokens[3].strip()
130 if not line_number_str.isdigit():
133 line_number = int(line_number_str)
135 if path and level and kind and message:
136 add_comment(path, line_number, level, kind, message)
140 # ERROR:CODE_INDENT: code indent should use tabs where possible
142 level, kind, message = line.split(':', 2)
144 level, kind, message = None, None, None
146 if level != 'ERROR' and level != 'WARNING':
147 level, kind, message = None, None, None
150 def review_input_and_score(path_line_comments, warning_count):
152 Convert { PATH: { LINE: [COMMENT, ...] }, ... }, [11] to a gerrit
153 ReviewInput() and score
157 for path, line_comments in path_line_comments.iteritems():
159 for line, comment_list in line_comments.iteritems():
160 message = '\n'.join(comment_list)
161 path_comments.append({'line': line, 'message': message})
162 review_comments[path] = path_comments
164 if warning_count[0] > 0:
169 if USE_CODE_REVIEW_SCORE:
170 code_review_score = score
172 code_review_score = 0
176 'message': ('%d style warning(s).\nFor more details please see %s' %
177 (warning_count[0], STYLE_LINK)),
179 'Code-Review': code_review_score
181 'comments': review_comments,
186 'message': 'Looks good to me.',
188 'Code-Review': code_review_score
196 return long(time.time())
199 class Reviewer(object):
201 * Poll gerrit instance for updates to changes matching project and branch.
202 * Pipe new patches through checkpatch.
203 * Convert checkpatch output to gerrit ReviewInput().
204 * Post ReviewInput() to gerrit instance.
205 * Track reviewed revisions in history_path.
207 def __init__(self, host, project, branch, username, password, history_path):
209 self.project = project
211 self.auth = requests.auth.HTTPDigestAuth(username, password)
212 self.logger = logging.getLogger(__name__)
213 self.history_path = history_path
214 self.history_mode = 'rw'
217 self.post_enabled = True
218 self.post_interval = 10
219 self.update_interval = 300
220 self.request_timeout = 60
222 def _debug(self, msg, *args):
224 self.logger.debug(msg, *args)
226 def _error(self, msg, *args):
228 self.logger.error(msg, *args)
230 def _url(self, path):
232 return 'http://' + self.host + '/a' + path
234 def _get(self, path):
236 GET path return Response.
238 url = self._url(path)
240 res = requests.get(url, auth=self.auth,
241 timeout=self.request_timeout)
242 except Exception as exc:
243 self._error("cannot GET '%s': exception = %s", url, str(exc))
246 if res.status_code != requests.codes.ok:
247 self._error("cannot GET '%s': reason = %s, status_code = %d",
248 url, res.reason, res.status_code)
253 def _post(self, path, obj):
255 POST json(obj) to path, return True on success.
257 url = self._url(path)
258 data = json.dumps(obj)
259 if not self.post_enabled:
260 self._debug("_post: disabled: url = '%s', data = '%s'", url, data)
264 res = requests.post(url, data=data,
265 headers={'Content-Type': 'application/json'},
266 auth=self.auth, timeout=self.request_timeout)
267 except Exception as exc:
268 self._error("cannot POST '%s': exception = %s", url, str(exc))
271 if res.status_code != requests.codes.ok:
272 self._error("cannot POST '%s': reason = %s, status_code = %d",
273 url, res.reason, res.status_code)
278 def load_history(self):
280 Load review history from history_path containing lines of the form:
281 EPOCH FULL_CHANGE_ID REVISION SCORE
282 1394536722 fs%2Flustre-release~master~I5cc6c23... 00e2cc75... 1
284 1394537033 fs%2Flustre-release~master~I10be8e9... 44f7b504... 1
289 if 'r' in self.history_mode:
290 with open(self.history_path) as history_file:
291 for line in history_file:
292 epoch, change_id, revision, score = line.split()
294 self.timestamp = long(float(epoch))
296 self.history[change_id + ' ' + revision] = score
298 self._debug("load_history: history size = %d, timestamp = %d",
299 len(self.history), self.timestamp)
301 def write_history(self, change_id, revision, score, epoch=-1):
303 Add review record to history dict and file.
306 self.history[change_id + ' ' + revision] = score
309 epoch = self.timestamp
311 if 'w' in self.history_mode:
312 with open(self.history_path, 'a') as history_file:
313 print >> history_file, epoch, change_id, revision, score
315 def in_history(self, change_id, revision):
317 Return True if change_id/revision was already reviewed.
319 return change_id + ' ' + revision in self.history
321 def get_change_by_id(self, change_id):
323 GET one change by id.
325 path = ('/changes/' + urllib.quote(self.project, safe='') + '~' +
326 urllib.quote(self.branch, safe='') + '~' + change_id +
327 '?o=CURRENT_REVISION')
328 res = self._get(path)
332 # Gerrit uses " )]}'" to guard against XSSI.
333 return json.loads(res.content[5:])
335 def get_changes(self, query):
337 GET a list of ChangeInfo()s for all changes matching query.
339 {'status':'open', '-age':'60m'} =>
340 GET /changes/?q=project:...+status:open+-age:60m&o=CURRENT_REVISION =>
344 project = query.get('project', self.project)
345 query['project'] = urllib.quote(project, safe='')
346 branch = query.get('branch', self.branch)
347 query['branch'] = urllib.quote(branch, safe='')
348 path = ('/changes/?q=' +
349 '+'.join(k + ':' + v for k, v in query.iteritems()) +
350 '&o=CURRENT_REVISION')
351 res = self._get(path)
355 # Gerrit uses " )]}'" to guard against XSSI.
356 return json.loads(res.content[5:])
358 def decode_patch(self, content):
360 Decode gerrit's idea of base64.
362 The base64 encoded patch returned by gerrit isn't always
363 padded correctly according to b64decode. Don't know why. Work
364 around this by appending more '=' characters or truncating the
365 content until it decodes. But do try the unmodified content
368 for i in (0, 1, 2, 3, -1, -2, -3):
370 padded_content = content + (i * '=')
372 padded_content = content[:i]
375 return base64.b64decode(padded_content)
376 except TypeError as exc:
377 self._debug("decode_patch: len = %d, exception = %s",
378 len(padded_content), str(exc))
382 def get_patch(self, change, revision='current'):
384 GET and decode the (current) patch for change.
386 path = '/changes/' + change['id'] + '/revisions/' + revision + '/patch'
387 self._debug("get_patch: path = '%s'", path)
388 res = self._get(path)
392 self._debug("get_patch: len(content) = %d, content = '%s...'",
393 len(res.content), res.content[:20])
395 return self.decode_patch(res.content)
397 def post_review(self, change, revision, review_input):
399 POST review_input for the given revision of change.
401 path = '/changes/' + change['id'] + '/revisions/' + revision + '/review'
402 self._debug("post_review: path = '%s'", path)
403 return self._post(path, review_input)
405 def check_patch(self, patch):
407 Run each script in CHECKPATCH_PATHS on patch, return a
408 ReviewInput() and score.
410 path_line_comments = {}
413 for path in CHECKPATCH_PATHS:
414 pipe = subprocess.Popen([path, '--show-types', '-'],
415 stdin=subprocess.PIPE,
416 stdout=subprocess.PIPE,
417 stderr=subprocess.PIPE)
418 out, err = pipe.communicate(patch)
419 self._debug("check_patch: path = %s, out = '%s...', err = '%s...'",
420 path, out[:80], err[:80])
421 parse_checkpatch_output(out, path_line_comments, warning_count)
423 return review_input_and_score(path_line_comments, warning_count)
425 def change_needs_review(self, change):
427 * Bail if the change isn't open (status is not 'NEW').
428 * Bail if we've already reviewed the current revision.
430 status = change.get('status')
432 self._debug("change_needs_review: status = %s", status)
435 current_revision = change.get('current_revision')
436 self._debug("change_needs_review: current_revision = '%s'",
438 if not current_revision:
441 # Have we already checked this revision?
442 if self.in_history(change['id'], current_revision):
443 self._debug("change_needs_review: already reviewed")
448 def review_change(self, change):
450 Review the current revision of change.
451 * Pipe the patch through checkpatch(es).
452 * Save results to review history.
453 * POST review to gerrit.
455 self._debug("review_change: change = %s, subject = '%s'",
456 change['id'], change.get('subject', ''))
458 current_revision = change.get('current_revision')
459 self._debug("change_needs_review: current_revision = '%s'",
461 if not current_revision:
464 patch = self.get_patch(change, current_revision)
466 self._debug("review_change: no patch")
469 review_input, score = self.check_patch(patch)
470 self._debug("review_change: score = %d", score)
471 self.write_history(change['id'], current_revision, score)
472 self.post_review(change, current_revision, review_input)
476 GET recently updated changes and review as needed.
478 new_timestamp = _now()
479 age = new_timestamp - self.timestamp + 60 * 60 # 1h padding
480 self._debug("update: age = %d", age)
482 open_changes = self.get_changes({'status':'open',
483 '-age':str(age) + 's'})
484 self._debug("update: got %d open_changes", len(open_changes))
486 for change in open_changes:
487 if self.change_needs_review(change):
488 self.review_change(change)
489 # Don't POST more than every post_interval seconds.
490 time.sleep(self.post_interval)
492 self.timestamp = new_timestamp
493 self.write_history('-', '-', 0)
497 * Load review history.
498 * Call update() every poll_interval seconds.
501 if self.timestamp <= 0:
506 time.sleep(self.update_interval)
511 logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
513 with open(GERRIT_AUTH_PATH) as auth_file:
514 auth = json.load(auth_file)
515 username = auth[GERRIT_HOST]['gerrit/http']['username']
516 password = auth[GERRIT_HOST]['gerrit/http']['password']
518 reviewer = Reviewer(GERRIT_HOST, GERRIT_PROJECT, GERRIT_BRANCH,
519 username, password, REVIEW_HISTORY_PATH)
523 if __name__ == "__main__":