1 #!/usr/bin/env python3
2 """Check proposed changes for common issues."""
3 import re
4 import sys
5 import shutil
6 import os.path
7 import subprocess
8 import sysconfig
9
10 import reindent
11 import untabify
12
13
14 def get_python_source_dir():
15 src_dir = sysconfig.get_config_var('abs_srcdir')
16 if not src_dir:
17 src_dir = sysconfig.get_config_var('srcdir')
18 return os.path.abspath(src_dir)
19
20
21 # Excluded directories which are copies of external libraries:
22 # don't check their coding style
23 EXCLUDE_DIRS = [os.path.join('Modules', '_ctypes', 'libffi_osx'),
24 os.path.join('Modules', '_ctypes', 'libffi_msvc'),
25 os.path.join('Modules', '_decimal', 'libmpdec'),
26 os.path.join('Modules', 'expat'),
27 os.path.join('Modules', 'zlib')]
28 SRCDIR = get_python_source_dir()
29
30
31 def n_files_str(count):
32 """Return 'N file(s)' with the proper plurality on 'file'."""
33 s = "s" if count != 1 else ""
34 return f"{count} file{s}"
35
36
37 def status(message, modal=False, info=None):
38 """Decorator to output status info to stdout."""
39 def decorated_fxn(fxn):
40 def call_fxn(*args, **kwargs):
41 sys.stdout.write(message + ' ... ')
42 sys.stdout.flush()
43 result = fxn(*args, **kwargs)
44 if not modal and not info:
45 print("done")
46 elif info:
47 print(info(result))
48 else:
49 print("yes" if result else "NO")
50 return result
51 return call_fxn
52 return decorated_fxn
53
54
55 def get_git_branch():
56 """Get the symbolic name for the current git branch"""
57 cmd = "git rev-parse --abbrev-ref HEAD".split()
58 try:
59 return subprocess.check_output(cmd,
60 stderr=subprocess.DEVNULL,
61 cwd=SRCDIR,
62 encoding='UTF-8')
63 except subprocess.CalledProcessError:
64 return None
65
66
67 def get_git_upstream_remote():
68 """Get the remote name to use for upstream branches
69
70 Uses "upstream" if it exists, "origin" otherwise
71 """
72 cmd = "git remote get-url upstream".split()
73 try:
74 subprocess.check_output(cmd,
75 stderr=subprocess.DEVNULL,
76 cwd=SRCDIR,
77 encoding='UTF-8')
78 except subprocess.CalledProcessError:
79 return "origin"
80 return "upstream"
81
82
83 def get_git_remote_default_branch(remote_name):
84 """Get the name of the default branch for the given remote
85
86 It is typically called 'main', but may differ
87 """
88 cmd = f"git remote show {remote_name}".split()
89 env = os.environ.copy()
90 env['LANG'] = 'C'
91 try:
92 remote_info = subprocess.check_output(cmd,
93 stderr=subprocess.DEVNULL,
94 cwd=SRCDIR,
95 encoding='UTF-8',
96 env=env)
97 except subprocess.CalledProcessError:
98 return None
99 for line in remote_info.splitlines():
100 if "HEAD branch:" in line:
101 base_branch = line.split(":")[1].strip()
102 return base_branch
103 return None
104
105
106 @status("Getting base branch for PR",
107 info=lambda x: x if x is not None else "not a PR branch")
108 def get_base_branch():
109 if not os.path.exists(os.path.join(SRCDIR, '.git')):
110 # Not a git checkout, so there's no base branch
111 return None
112 upstream_remote = get_git_upstream_remote()
113 version = sys.version_info
114 if version.releaselevel == 'alpha':
115 base_branch = get_git_remote_default_branch(upstream_remote)
116 else:
117 base_branch = "{0.major}.{0.minor}".format(version)
118 this_branch = get_git_branch()
119 if this_branch is None or this_branch == base_branch:
120 # Not on a git PR branch, so there's no base branch
121 return None
122 return upstream_remote + "/" + base_branch
123
124
125 @status("Getting the list of files that have been added/changed",
126 info=lambda x: n_files_str(len(x)))
127 def changed_files(base_branch=None):
128 """Get the list of changed or added files from git."""
129 if os.path.exists(os.path.join(SRCDIR, '.git')):
130 # We just use an existence check here as:
131 # directory = normal git checkout/clone
132 # file = git worktree directory
133 if base_branch:
134 cmd = 'git diff --name-status ' + base_branch
135 else:
136 cmd = 'git status --porcelain'
137 filenames = []
138 with subprocess.Popen(cmd.split(),
139 stdout=subprocess.PIPE,
140 cwd=SRCDIR) as st:
141 if st.wait() != 0:
142 sys.exit(f'error running {cmd}')
143 for line in st.stdout:
144 line = line.decode().rstrip()
145 status_text, filename = line.split(maxsplit=1)
146 status = set(status_text)
147 # modified, added or unmerged files
148 if not status.intersection('MAU'):
149 continue
150 if ' -> ' in filename:
151 # file is renamed
152 filename = filename.split(' -> ', 2)[1].strip()
153 filenames.append(filename)
154 else:
155 sys.exit('need a git checkout to get modified files')
156
157 filenames2 = []
158 for filename in filenames:
159 # Normalize the path to be able to match using .startswith()
160 filename = os.path.normpath(filename)
161 if any(filename.startswith(path) for path in EXCLUDE_DIRS):
162 # Exclude the file
163 continue
164 filenames2.append(filename)
165
166 return filenames2
167
168
169 def report_modified_files(file_paths):
170 count = len(file_paths)
171 if count == 0:
172 return n_files_str(count)
173 else:
174 lines = [f"{n_files_str(count)}:"]
175 for path in file_paths:
176 lines.append(f" {path}")
177 return "\n".join(lines)
178
179
180 @status("Fixing Python file whitespace", info=report_modified_files)
181 def normalize_whitespace(file_paths):
182 """Make sure that the whitespace for .py files have been normalized."""
183 reindent.makebackup = False # No need to create backups.
184 fixed = [path for path in file_paths if path.endswith('.py') and
185 reindent.check(os.path.join(SRCDIR, path))]
186 return fixed
187
188
189 @status("Fixing C file whitespace", info=report_modified_files)
190 def normalize_c_whitespace(file_paths):
191 """Report if any C files """
192 fixed = []
193 for path in file_paths:
194 abspath = os.path.join(SRCDIR, path)
195 with open(abspath, 'r') as f:
196 if '\t' not in f.read():
197 continue
198 untabify.process(abspath, 8, verbose=False)
199 fixed.append(path)
200 return fixed
201
202
203 @status("Docs modified", modal=True)
204 def docs_modified(file_paths):
205 """Report if any file in the Doc directory has been changed."""
206 return bool(file_paths)
207
208
209 @status("Misc/ACKS updated", modal=True)
210 def credit_given(file_paths):
211 """Check if Misc/ACKS has been changed."""
212 return os.path.join('Misc', 'ACKS') in file_paths
213
214
215 @status("Misc/NEWS.d updated with `blurb`", modal=True)
216 def reported_news(file_paths):
217 """Check if Misc/NEWS.d has been changed."""
218 return any(p.startswith(os.path.join('Misc', 'NEWS.d', 'next'))
219 for p in file_paths)
220
221
222 @status("configure regenerated", modal=True, info=str)
223 def regenerated_configure(file_paths):
224 """Check if configure has been regenerated."""
225 if 'configure.ac' in file_paths:
226 return "yes" if 'configure' in file_paths else "no"
227 else:
228 return "not needed"
229
230
231 @status("pyconfig.h.in regenerated", modal=True, info=str)
232 def regenerated_pyconfig_h_in(file_paths):
233 """Check if pyconfig.h.in has been regenerated."""
234 if 'configure.ac' in file_paths:
235 return "yes" if 'pyconfig.h.in' in file_paths else "no"
236 else:
237 return "not needed"
238
239
240 def ci(pull_request):
241 if pull_request == 'false':
242 print('Not a pull request; skipping')
243 return
244 base_branch = get_base_branch()
245 file_paths = changed_files(base_branch)
246 python_files = [fn for fn in file_paths if fn.endswith('.py')]
247 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
248 fixed = []
249 fixed.extend(normalize_whitespace(python_files))
250 fixed.extend(normalize_c_whitespace(c_files))
251 if not fixed:
252 print('No whitespace issues found')
253 else:
254 count = len(fixed)
255 print(f'Please fix the {n_files_str(count)} with whitespace issues')
256 print('(on Unix you can run `make patchcheck` to make the fixes)')
257 sys.exit(1)
258
259
260 def main():
261 base_branch = get_base_branch()
262 file_paths = changed_files(base_branch)
263 python_files = [fn for fn in file_paths if fn.endswith('.py')]
264 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
265 doc_files = [fn for fn in file_paths if fn.startswith('Doc') and
266 fn.endswith(('.rst', '.inc'))]
267 misc_files = {p for p in file_paths if p.startswith('Misc')}
268 # PEP 8 whitespace rules enforcement.
269 normalize_whitespace(python_files)
270 # C rules enforcement.
271 normalize_c_whitespace(c_files)
272 # Docs updated.
273 docs_modified(doc_files)
274 # Misc/ACKS changed.
275 credit_given(misc_files)
276 # Misc/NEWS changed.
277 reported_news(misc_files)
278 # Regenerated configure, if necessary.
279 regenerated_configure(file_paths)
280 # Regenerated pyconfig.h.in, if necessary.
281 regenerated_pyconfig_h_in(file_paths)
282
283 # Test suite run and passed.
284 if python_files or c_files:
285 end = " and check for refleaks?" if c_files else "?"
286 print()
287 print("Did you run the test suite" + end)
288
289
290 if __name__ == '__main__':
291 import argparse
292 parser = argparse.ArgumentParser(description=__doc__)
293 parser.add_argument('--ci',
294 help='Perform pass/fail checks')
295 args = parser.parse_args()
296 if args.ci:
297 ci(args.ci)
298 else:
299 main()