5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 2 only,
9 # as published by the Free Software Foundation.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License version 2 for more details (a copy is included
15 # in the LICENSE file that accompanied this code).
17 # You should have received a copy of the GNU General Public License
18 # version 2 along with this program; If not, see
19 # http://www.gnu.org/licenses/gpl-2.0.html
23 # Copyright (c) 2014, Intel Corporation.
25 # Author: John L. Hammond <john.hammond@intel.com>
28 Gerrit Checkpatch Reviewer Daemon
29 ~~~~~~ ~~~~~~~~~~ ~~~~~~~~ ~~~~~~
31 * Watch for new change revisions in a gerrit instance.
32 * Pass new revisions through checkpatch script.
33 * POST reviews back to gerrit based on checkpatch output.
46 def _getenv_list(key, default=None, sep=':'):
48 'PATH' => ['/bin', '/usr/bin', ...]
50 value = os.getenv(key)
54 return value.split(sep)
56 GERRIT_HOST = os.getenv('GERRIT_HOST', 'review.whamcloud.com')
57 GERRIT_PROJECT = os.getenv('GERRIT_PROJECT', 'fs/lustre-release')
58 GERRIT_BRANCH = os.getenv('GERRIT_BRANCH', 'master')
59 GERRIT_AUTH_PATH = os.getenv('GERRIT_AUTH_PATH', 'GERRIT_AUTH')
60 GERRIT_CHANGE_NUMBER = os.getenv('GERRIT_CHANGE_NUMBER', None)
62 # GERRIT_AUTH should contain a single JSON dictionary of the form:
64 # "review.example.com": {
66 # "username": "example-checkpatch",
73 CHECKPATCH_PATHS = _getenv_list('CHECKPATCH_PATHS', ['checkpatch.pl'])
74 CHECKPATCH_ARGS = os.getenv('CHECKPATCH_ARGS','--show-types -').split(' ')
75 CHECKPATCH_IGNORED_FILES = _getenv_list('CHECKPATCH_IGNORED_FILES', [
76 'lustre/contrib/wireshark/packet-lustre.c',
77 'lustre/ptlrpc/wiretest.c',
78 'lustre/utils/wiretest.c',
80 CHECKPATCH_IGNORED_KINDS = _getenv_list('CHECKPATCH_IGNORED_KINDS', [
84 REVIEW_HISTORY_PATH = os.getenv('REVIEW_HISTORY_PATH', 'REVIEW_HISTORY')
85 STYLE_LINK = os.getenv('STYLE_LINK',
86 'http://wiki.lustre.org/Lustre_Coding_Style_Guidelines')
88 USE_CODE_REVIEW_SCORE = False
90 def parse_checkpatch_output(out, path_line_comments, warning_count):
92 Parse string output out of CHECKPATCH into path_line_comments.
93 Increment warning_count[0] for each warning.
95 path_line_comments is { PATH: { LINE: [COMMENT, ...] }, ... }.
97 def add_comment(path, line, level, kind, message):
99 logging.debug("add_comment %s %d %s %s '%s'",
100 path, line, level, kind, message)
101 if kind in CHECKPATCH_IGNORED_KINDS:
104 for pattern in CHECKPATCH_IGNORED_FILES:
105 if fnmatch.fnmatch(path, pattern):
108 path_comments = path_line_comments.setdefault(path, {})
109 line_comments = path_comments.setdefault(line, [])
110 line_comments.append('(style) %s\n' % message)
111 warning_count[0] += 1
113 level = None # 'ERROR', 'WARNING'
114 kind = None # 'CODE_INDENT', 'LEADING_SPACE', ...
115 message = None # 'code indent should use tabs where possible'
117 for line in out.splitlines():
118 # ERROR:CODE_INDENT: code indent should use tabs where possible
119 # #404: FILE: lustre/liblustre/dir.c:103:
120 # + op_data.op_hash_offset = hash_x_index(page->index, 0);$
123 level, kind, message = None, None, None
125 # '#404: FILE: lustre/liblustre/dir.c:103:'
126 tokens = line.split(':', 5)
127 if len(tokens) != 5 or tokens[1] != ' FILE':
130 path = tokens[2].strip()
131 line_number_str = tokens[3].strip()
132 if not line_number_str.isdigit():
135 line_number = int(line_number_str)
137 if path and level and kind and message:
138 add_comment(path, line_number, level, kind, message)
142 # ERROR:CODE_INDENT: code indent should use tabs where possible
144 level, kind, message = line.split(':', 2)
146 level, kind, message = None, None, None
148 if level != 'ERROR' and level != 'WARNING':
149 level, kind, message = None, None, None
152 def review_input_and_score(path_line_comments, warning_count):
154 Convert { PATH: { LINE: [COMMENT, ...] }, ... }, [11] to a gerrit
155 ReviewInput() and score
159 for path, line_comments in path_line_comments.iteritems():
161 for line, comment_list in line_comments.iteritems():
162 message = '\n'.join(comment_list)
163 path_comments.append({'line': line, 'message': message})
164 review_comments[path] = path_comments
166 if warning_count[0] > 0:
171 if USE_CODE_REVIEW_SCORE:
172 code_review_score = score
174 code_review_score = 0
178 'message': ('%d style warning(s).\nFor more details please see %s' %
179 (warning_count[0], STYLE_LINK)),
181 'Code-Review': code_review_score
183 'comments': review_comments,
188 'message': 'Looks good to me.',
190 'Code-Review': code_review_score
198 return long(time.time())
201 class Reviewer(object):
203 * Poll gerrit instance for updates to changes matching project and branch.
204 * Pipe new patches through checkpatch.
205 * Convert checkpatch output to gerrit ReviewInput().
206 * Post ReviewInput() to gerrit instance.
207 * Track reviewed revisions in history_path.
209 def __init__(self, host, project, branch, username, password, history_path):
211 self.project = project
213 self.auth = requests.auth.HTTPDigestAuth(username, password)
214 self.logger = logging.getLogger(__name__)
215 self.history_path = history_path
216 self.history_mode = 'rw'
219 self.post_enabled = True
220 self.post_interval = 10
221 self.update_interval = 300
222 self.request_timeout = 60
224 def _debug(self, msg, *args):
226 self.logger.debug(msg, *args)
228 def _error(self, msg, *args):
230 self.logger.error(msg, *args)
232 def _url(self, path):
234 return 'http://' + self.host + '/a' + path
236 def _get(self, path):
238 GET path return Response.
240 url = self._url(path)
242 res = requests.get(url, auth=self.auth,
243 timeout=self.request_timeout)
244 except Exception as exc:
245 self._error("cannot GET '%s': exception = %s", url, str(exc))
248 if res.status_code != requests.codes.ok:
249 self._error("cannot GET '%s': reason = %s, status_code = %d",
250 url, res.reason, res.status_code)
255 def _post(self, path, obj):
257 POST json(obj) to path, return True on success.
259 url = self._url(path)
260 data = json.dumps(obj)
261 if not self.post_enabled:
262 self._debug("_post: disabled: url = '%s', data = '%s'", url, data)
266 res = requests.post(url, data=data,
267 headers={'Content-Type': 'application/json'},
268 auth=self.auth, timeout=self.request_timeout)
269 except Exception as exc:
270 self._error("cannot POST '%s': exception = %s", url, str(exc))
273 if res.status_code != requests.codes.ok:
274 self._error("cannot POST '%s': reason = %s, status_code = %d",
275 url, res.reason, res.status_code)
280 def load_history(self):
282 Load review history from history_path containing lines of the form:
283 EPOCH FULL_CHANGE_ID REVISION SCORE
284 1394536722 fs%2Flustre-release~master~I5cc6c23... 00e2cc75... 1
286 1394537033 fs%2Flustre-release~master~I10be8e9... 44f7b504... 1
291 if 'r' in self.history_mode:
292 with open(self.history_path) as history_file:
293 for line in history_file:
294 epoch, change_id, revision, score = line.split()
296 self.timestamp = long(float(epoch))
298 self.history[change_id + ' ' + revision] = score
300 self._debug("load_history: history size = %d, timestamp = %d",
301 len(self.history), self.timestamp)
303 def write_history(self, change_id, revision, score, epoch=-1):
305 Add review record to history dict and file.
308 self.history[change_id + ' ' + revision] = score
311 epoch = self.timestamp
313 if 'w' in self.history_mode:
314 with open(self.history_path, 'a') as history_file:
315 print >> history_file, epoch, change_id, revision, score
317 def in_history(self, change_id, revision):
319 Return True if change_id/revision was already reviewed.
321 return change_id + ' ' + revision in self.history
323 def get_change_by_id(self, change_id):
325 GET one change by id.
327 path = ('/changes/' + urllib.quote(self.project, safe='') + '~' +
328 urllib.quote(self.branch, safe='') + '~' + change_id +
329 '?o=CURRENT_REVISION')
330 res = self._get(path)
334 # Gerrit uses " )]}'" to guard against XSSI.
335 return json.loads(res.content[5:])
337 def get_changes(self, query):
339 GET a list of ChangeInfo()s for all changes matching query.
341 {'status':'open', '-age':'60m'} =>
342 GET /changes/?q=project:...+status:open+-age:60m&o=CURRENT_REVISION =>
346 project = query.get('project', self.project)
347 query['project'] = urllib.quote(project, safe='')
348 branch = query.get('branch', self.branch)
349 query['branch'] = urllib.quote(branch, safe='')
350 path = ('/changes/?q=' +
351 '+'.join(k + ':' + v for k, v in query.iteritems()) +
352 '&o=CURRENT_REVISION')
353 res = self._get(path)
357 # Gerrit uses " )]}'" to guard against XSSI.
358 return json.loads(res.content[5:])
360 def decode_patch(self, content):
362 Decode gerrit's idea of base64.
364 The base64 encoded patch returned by gerrit isn't always
365 padded correctly according to b64decode. Don't know why. Work
366 around this by appending more '=' characters or truncating the
367 content until it decodes. But do try the unmodified content
370 for i in (0, 1, 2, 3, -1, -2, -3):
372 padded_content = content + (i * '=')
374 padded_content = content[:i]
377 return base64.b64decode(padded_content)
378 except TypeError as exc:
379 self._debug("decode_patch: len = %d, exception = %s",
380 len(padded_content), str(exc))
384 def get_patch(self, change, revision='current'):
386 GET and decode the (current) patch for change.
388 path = '/changes/' + change['id'] + '/revisions/' + revision + '/patch'
389 self._debug("get_patch: path = '%s'", path)
390 res = self._get(path)
394 self._debug("get_patch: len(content) = %d, content = '%s...'",
395 len(res.content), res.content[:20])
397 return self.decode_patch(res.content)
399 def post_review(self, change, revision, review_input):
401 POST review_input for the given revision of change.
403 path = '/changes/' + change['id'] + '/revisions/' + revision + '/review'
404 self._debug("post_review: path = '%s'", path)
405 return self._post(path, review_input)
407 def check_patch(self, patch):
409 Run each script in CHECKPATCH_PATHS on patch, return a
410 ReviewInput() and score.
412 path_line_comments = {}
415 for path in CHECKPATCH_PATHS:
416 pipe = subprocess.Popen([path] + CHECKPATCH_ARGS,
417 stdin=subprocess.PIPE,
418 stdout=subprocess.PIPE,
419 stderr=subprocess.PIPE)
420 out, err = pipe.communicate(patch)
421 self._debug("check_patch: path = %s, out = '%s...', err = '%s...'",
422 path, out[:80], err[:80])
423 parse_checkpatch_output(out, path_line_comments, warning_count)
425 return review_input_and_score(path_line_comments, warning_count)
427 def change_needs_review(self, change):
429 * Bail if the change isn't open (status is not 'NEW').
430 * Bail if we've already reviewed the current revision.
432 status = change.get('status')
434 self._debug("change_needs_review: status = %s", status)
437 current_revision = change.get('current_revision')
438 self._debug("change_needs_review: current_revision = '%s'",
440 if not current_revision:
443 # Have we already checked this revision?
444 if self.in_history(change['id'], current_revision):
445 self._debug("change_needs_review: already reviewed")
450 def review_change(self, change):
452 Review the current revision of change.
453 * Pipe the patch through checkpatch(es).
454 * Save results to review history.
455 * POST review to gerrit.
457 self._debug("review_change: change = %s, subject = '%s'",
458 change['id'], change.get('subject', ''))
460 current_revision = change.get('current_revision')
461 self._debug("change_needs_review: current_revision = '%s'",
463 if not current_revision:
466 patch = self.get_patch(change, current_revision)
468 self._debug("review_change: no patch")
471 review_input, score = self.check_patch(patch)
472 self._debug("review_change: score = %d", score)
473 self.write_history(change['id'], current_revision, score)
474 self.post_review(change, current_revision, review_input)
478 GET recently updated changes and review as needed.
480 new_timestamp = _now()
481 age = new_timestamp - self.timestamp + 60 * 60 # 1h padding
482 self._debug("update: age = %d", age)
484 open_changes = self.get_changes({'status':'open',
485 '-age':str(age) + 's'})
486 self._debug("update: got %d open_changes", len(open_changes))
488 for change in open_changes:
489 if self.change_needs_review(change):
490 self.review_change(change)
491 # Don't POST more than every post_interval seconds.
492 time.sleep(self.post_interval)
494 self.timestamp = new_timestamp
495 self.write_history('-', '-', 0)
497 def update_single_change(self, change):
501 open_changes = self.get_changes({'status':'open',
503 self._debug("update: got %d open_changes", len(open_changes))
505 for change in open_changes:
506 if self.change_needs_review(change):
507 self.review_change(change)
511 * Load review history.
512 * Call update() every poll_interval seconds.
515 if self.timestamp <= 0:
520 time.sleep(self.update_interval)
525 logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
527 with open(GERRIT_AUTH_PATH) as auth_file:
528 auth = json.load(auth_file)
529 username = auth[GERRIT_HOST]['gerrit/http']['username']
530 password = auth[GERRIT_HOST]['gerrit/http']['password']
532 reviewer = Reviewer(GERRIT_HOST, GERRIT_PROJECT, GERRIT_BRANCH,
533 username, password, REVIEW_HISTORY_PATH)
535 if GERRIT_CHANGE_NUMBER:
536 reviewer.update_single_change(GERRIT_CHANGE_NUMBER)
541 if __name__ == "__main__":