1 #!/usr/bin/env python3
2
3 # Copyright (C) 2020-2023 Free Software Foundation, Inc.
4 #
5 # This file is part of GCC.
6 #
7 # GCC is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3, or (at your option)
10 # any later version.
11 #
12 # GCC is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with GCC; see the file COPYING. If not, write to
19 # the Free Software Foundation, 51 Franklin Street, Fifth Floor,
20 # Boston, MA 02110-1301, USA.
21
22 # This script parses a .diff file generated with 'diff -up' or 'diff -cp'
23 # and adds a skeleton ChangeLog file to the file. It does not try to be
24 # too smart when parsing function names, but it produces a reasonable
25 # approximation.
26 #
27 # Author: Martin Liska <mliska@suse.cz>
28
29 import argparse
30 import datetime
31 import json
32 import os
33 import re
34 import subprocess
35 import sys
36 from itertools import takewhile
37
38 import requests
39
40 from unidiff import PatchSet
41
42 LINE_LIMIT = 100
43 TAB_WIDTH = 8
44 CO_AUTHORED_BY_PREFIX = 'co-authored-by: '
45
46 pr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<pr>PR [a-z+-]+\/[0-9]+)')
47 prnum_regex = re.compile(r'PR (?P<comp>[a-z+-]+)/(?P<num>[0-9]+)')
48 dr_regex = re.compile(r'(\/(\/|\*)|[Cc*!])\s+(?P<dr>DR [0-9]+)')
49 dg_regex = re.compile(r'{\s+dg-(error|warning)')
50 pr_filename_regex = re.compile(r'(^|[\W_])[Pp][Rr](?P<pr>\d{4,})')
51 identifier_regex = re.compile(r'^([a-zA-Z0-9_#].*)')
52 comment_regex = re.compile(r'^\/\*')
53 struct_regex = re.compile(r'^(class|struct|union|enum)\s+'
54 r'(GTY\(.*\)\s+)?([a-zA-Z0-9_]+)')
55 macro_regex = re.compile(r'#\s*(define|undef)\s+([a-zA-Z0-9_]+)')
56 super_macro_regex = re.compile(r'^DEF[A-Z0-9_]+\s*\(([a-zA-Z0-9_]+)')
57 fn_regex = re.compile(r'([a-zA-Z_][^()\s]*)\s*\([^*]')
58 template_and_param_regex = re.compile(r'<[^<>]*>')
59 md_def_regex = re.compile(r'\(define.*\s+"(.*)"')
60 bugzilla_url = 'https://gcc.gnu.org/bugzilla/rest.cgi/bug?id=%s&' \
61 'include_fields=summary,component'
62
63 function_extensions = {'.c', '.cpp', '.C', '.cc', '.h', '.inc', '.def', '.md'}
64
65 # NB: Makefile.in isn't listed as it's not always generated.
66 generated_files = {'aclocal.m4', 'config.h.in', 'configure'}
67
68 help_message = """\
69 Generate ChangeLog template for PATCH.
70 PATCH must be generated using diff(1)'s -up or -cp options
71 (or their equivalent in git).
72 """
73
74 script_folder = os.path.realpath(__file__)
75 root = os.path.dirname(os.path.dirname(script_folder))
76
77
78 def find_changelog(path):
79 folder = os.path.split(path)[0]
80 while True:
81 if os.path.exists(os.path.join(root, folder, 'ChangeLog')):
82 return folder
83 folder = os.path.dirname(folder)
84 if folder == '':
85 return folder
86 raise AssertionError()
87
88
89 def extract_function_name(line):
90 if comment_regex.match(line):
91 return None
92 m = struct_regex.search(line)
93 if m:
94 # Struct declaration
95 return m.group(1) + ' ' + m.group(3)
96 m = macro_regex.search(line)
97 if m:
98 # Macro definition
99 return m.group(2)
100 m = super_macro_regex.search(line)
101 if m:
102 # Supermacro
103 return m.group(1)
104 m = fn_regex.search(line)
105 if m:
106 # Discard template and function parameters.
107 fn = m.group(1)
108 fn = re.sub(template_and_param_regex, '', fn)
109 return fn.rstrip()
110 return None
111
112
113 def try_add_function(functions, line):
114 fn = extract_function_name(line)
115 if fn and fn not in functions:
116 functions.append(fn)
117 return bool(fn)
118
119
120 def sort_changelog_files(changed_file):
121 return (changed_file.is_added_file, changed_file.is_removed_file)
122
123
124 def get_pr_titles(prs):
125 output = []
126 for idx, pr in enumerate(prs):
127 pr_id = pr.split('/')[-1]
128 r = requests.get(bugzilla_url % pr_id)
129 bugs = r.json()['bugs']
130 if len(bugs) == 1:
131 prs[idx] = 'PR %s/%s' % (bugs[0]['component'], pr_id)
132 out = '%s - %s\n' % (prs[idx], bugs[0]['summary'])
133 if out not in output:
134 output.append(out)
135 if output:
136 output.append('')
137 return '\n'.join(output)
138
139
140 def append_changelog_line(out, relative_path, text):
141 line = f'\t* {relative_path}:'
142 if len(line.replace('\t', ' ' * TAB_WIDTH) + ' ' + text) <= LINE_LIMIT:
143 out += f'{line} {text}\n'
144 else:
145 out += f'{line}\n'
146 out += f'\t{text}\n'
147 return out
148
149
150 def get_rel_path_if_prefixed(path, folder):
151 if path.startswith(folder):
152 return path[len(folder):].lstrip('/')
153 else:
154 return path
155
156
157 def generate_changelog(data, no_functions=False, fill_pr_titles=False,
158 additional_prs=None):
159 global prs
160 prs = []
161
162 changelogs = {}
163 changelog_list = []
164 out = ''
165 diff = PatchSet(data)
166
167 if additional_prs:
168 for apr in additional_prs:
169 if not apr.startswith('PR ') and '/' in apr:
170 apr = 'PR ' + apr
171 if apr not in prs:
172 prs.append(apr)
173 for file in diff:
174 # skip files that can't be parsed
175 if file.path == '/dev/null':
176 continue
177 changelog = find_changelog(file.path)
178 if changelog not in changelogs:
179 changelogs[changelog] = []
180 changelog_list.append(changelog)
181 changelogs[changelog].append(file)
182
183 # Extract PR entries from newly added tests
184 if 'testsuite' in file.path and file.is_added_file:
185 # Only search first ten lines as later lines may
186 # contains commented code which a note that it
187 # has not been tested due to a certain PR or DR.
188 this_file_prs = []
189 hunks = list(file)
190 if hunks:
191 for line in hunks[0][0:10]:
192 m = pr_regex.search(line.value)
193 if m:
194 pr = m.group('pr')
195 if pr not in prs:
196 prs.append(pr)
197 this_file_prs.append(pr.split('/')[-1])
198 else:
199 m = dr_regex.search(line.value)
200 if m:
201 dr = m.group('dr')
202 if dr not in prs:
203 prs.append(dr)
204 this_file_prs.append(dr.split('/')[-1])
205 elif dg_regex.search(line.value):
206 # Found dg-warning/dg-error line
207 break
208
209 # PR number in the file name
210 fname = os.path.basename(file.path)
211 m = pr_filename_regex.search(fname)
212 if m:
213 pr = m.group('pr')
214 pr2 = 'PR ' + pr
215 if pr not in this_file_prs and pr2 not in prs:
216 prs.append(pr2)
217
218 if fill_pr_titles:
219 out += get_pr_titles(prs)
220
221 # print list of PR entries before ChangeLog entries
222 if prs:
223 if not out:
224 out += '\n'
225 for pr in prs:
226 out += '\t%s\n' % pr
227 out += '\n'
228
229 # sort ChangeLog so that 'testsuite' is at the end
230 for changelog in sorted(changelog_list, key=lambda x: 'testsuite' in x):
231 files = changelogs[changelog]
232 out += '%s:\n' % os.path.join(changelog, 'ChangeLog')
233 out += '\n'
234 # new and deleted files should be at the end
235 for file in sorted(files, key=sort_changelog_files):
236 assert file.path.startswith(changelog)
237 in_tests = 'testsuite' in changelog or 'testsuite' in file.path
238 relative_path = get_rel_path_if_prefixed(file.path, changelog)
239 functions = []
240 if file.is_added_file:
241 msg = 'New test.' if in_tests else 'New file.'
242 out = append_changelog_line(out, relative_path, msg)
243 elif file.is_removed_file:
244 out = append_changelog_line(out, relative_path, 'Removed.')
245 elif hasattr(file, 'is_rename') and file.is_rename:
246 # A file can be theoretically moved to a location that
247 # belongs to a different ChangeLog. Let user fix it.
248 #
249 # Since unidiff 0.7.0, path.file == path.target_file[2:],
250 # it used to be path.source_file[2:]
251 relative_path = get_rel_path_if_prefixed(file.source_file[2:],
252 changelog)
253 out = append_changelog_line(out, relative_path, 'Moved to...')
254 new_path = get_rel_path_if_prefixed(file.target_file[2:],
255 changelog)
256 out += f'\t* {new_path}: ...here.\n'
257 elif os.path.basename(file.path) in generated_files:
258 out += '\t* %s: Regenerate.\n' % (relative_path)
259 append_changelog_line(out, relative_path, 'Regenerate.')
260 else:
261 if not no_functions:
262 for hunk in file:
263 # Do not add function names for testsuite files
264 extension = os.path.splitext(relative_path)[1]
265 if not in_tests and extension in function_extensions:
266 last_fn = None
267 modified_visited = False
268 success = False
269 for line in hunk:
270 m = identifier_regex.match(line.value)
271 if line.is_added or line.is_removed:
272 # special-case definition in .md files
273 m2 = md_def_regex.match(line.value)
274 if extension == '.md' and m2:
275 fn = m2.group(1)
276 if fn not in functions:
277 functions.append(fn)
278 last_fn = None
279 success = True
280
281 if not line.value.strip():
282 continue
283 modified_visited = True
284 if m and try_add_function(functions,
285 m.group(1)):
286 last_fn = None
287 success = True
288 elif line.is_context:
289 if last_fn and modified_visited:
290 try_add_function(functions, last_fn)
291 last_fn = None
292 modified_visited = False
293 success = True
294 elif m:
295 last_fn = m.group(1)
296 modified_visited = False
297 if not success:
298 try_add_function(functions,
299 hunk.section_header)
300 if functions:
301 out += '\t* %s (%s):\n' % (relative_path, functions[0])
302 for fn in functions[1:]:
303 out += '\t(%s):\n' % fn
304 else:
305 out += '\t* %s:\n' % relative_path
306 out += '\n'
307 return out
308
309
310 def update_copyright(data):
311 current_timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
312 username = subprocess.check_output('git config user.name', shell=True,
313 encoding='utf8').strip()
314 email = subprocess.check_output('git config user.email', shell=True,
315 encoding='utf8').strip()
316
317 changelogs = set()
318 diff = PatchSet(data)
319
320 for file in diff:
321 changelog = os.path.join(find_changelog(file.path), 'ChangeLog')
322 if changelog not in changelogs:
323 changelogs.add(changelog)
324 with open(changelog) as f:
325 content = f.read()
326 with open(changelog, 'w+') as f:
327 f.write(f'{current_timestamp} {username} <{email}>\n\n')
328 f.write('\tUpdate copyright years.\n\n')
329 f.write(content)
330
331
332 def skip_line_in_changelog(line):
333 if line.lower().startswith(CO_AUTHORED_BY_PREFIX) or line.startswith('#'):
334 return False
335 return True
336
337
338 if __name__ == '__main__':
339 extra_args = os.getenv('GCC_MKLOG_ARGS')
340 if extra_args:
341 sys.argv += json.loads(extra_args)
342
343 parser = argparse.ArgumentParser(description=help_message)
344 parser.add_argument('input', nargs='?',
345 help='Patch file (or missing, read standard input)')
346 parser.add_argument('-b', '--pr-numbers', action='store',
347 type=lambda arg: arg.split(','), nargs='?',
348 help='Add the specified PRs (comma separated)')
349 parser.add_argument('-s', '--no-functions', action='store_true',
350 help='Do not generate function names in ChangeLogs')
351 parser.add_argument('-p', '--fill-up-bug-titles', action='store_true',
352 help='Download title of mentioned PRs')
353 parser.add_argument('-d', '--directory',
354 help='Root directory where to search for ChangeLog '
355 'files')
356 parser.add_argument('-c', '--changelog',
357 help='Append the ChangeLog to a git commit message '
358 'file')
359 parser.add_argument('--update-copyright', action='store_true',
360 help='Update copyright in ChangeLog files')
361 args = parser.parse_args()
362 if args.input == '-':
363 args.input = None
364 if args.directory:
365 root = args.directory
366
367 data = open(args.input, newline='\n') if args.input else sys.stdin
368 if args.update_copyright:
369 update_copyright(data)
370 else:
371 output = generate_changelog(data, args.no_functions,
372 args.fill_up_bug_titles, args.pr_numbers)
373 if args.changelog:
374 lines = open(args.changelog).read().split('\n')
375 start = list(takewhile(skip_line_in_changelog, lines))
376 end = lines[len(start):]
377 with open(args.changelog, 'w') as f:
378 if not start or not start[0]:
379 if len(prs) == 1:
380 # initial commit subject line 'component: [PRnnnnn]'
381 m = prnum_regex.match(prs[0])
382 if m:
383 title = f'{m.group("comp")}: [PR{m.group("num")}]'
384 start.insert(0, title)
385 if start:
386 # append empty line
387 if start[-1] != '':
388 start.append('')
389 else:
390 # append 2 empty lines
391 start = 2 * ['']
392 f.write('\n'.join(start))
393 f.write('\n')
394 f.write(output)
395 f.write('\n'.join(end))
396 else:
397 print(output, end='')