python (3.11.7)
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (C) 2013 Vinay Sajip.
4 # Licensed to the Python Software Foundation under a contributor agreement.
5 # See LICENSE.txt and CONTRIBUTORS.txt.
6 #
7 import hashlib
8 import logging
9 import os
10 import shutil
11 import subprocess
12 import tempfile
13 try:
14 from threading import Thread
15 except ImportError: # pragma: no cover
16 from dummy_threading import Thread
17
18 from . import DistlibException
19 from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,
20 urlparse, build_opener, string_types)
21 from .util import zip_dir, ServerProxy
22
23 logger = logging.getLogger(__name__)
24
25 DEFAULT_INDEX = 'https://pypi.org/pypi'
26 DEFAULT_REALM = 'pypi'
27
28 class ESC[4;38;5;81mPackageIndex(ESC[4;38;5;149mobject):
29 """
30 This class represents a package index compatible with PyPI, the Python
31 Package Index.
32 """
33
34 boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$'
35
36 def __init__(self, url=None):
37 """
38 Initialise an instance.
39
40 :param url: The URL of the index. If not specified, the URL for PyPI is
41 used.
42 """
43 self.url = url or DEFAULT_INDEX
44 self.read_configuration()
45 scheme, netloc, path, params, query, frag = urlparse(self.url)
46 if params or query or frag or scheme not in ('http', 'https'):
47 raise DistlibException('invalid repository: %s' % self.url)
48 self.password_handler = None
49 self.ssl_verifier = None
50 self.gpg = None
51 self.gpg_home = None
52 with open(os.devnull, 'w') as sink:
53 # Use gpg by default rather than gpg2, as gpg2 insists on
54 # prompting for passwords
55 for s in ('gpg', 'gpg2'):
56 try:
57 rc = subprocess.check_call([s, '--version'], stdout=sink,
58 stderr=sink)
59 if rc == 0:
60 self.gpg = s
61 break
62 except OSError:
63 pass
64
65 def _get_pypirc_command(self):
66 """
67 Get the distutils command for interacting with PyPI configurations.
68 :return: the command.
69 """
70 from .util import _get_pypirc_command as cmd
71 return cmd()
72
73 def read_configuration(self):
74 """
75 Read the PyPI access configuration as supported by distutils. This populates
76 ``username``, ``password``, ``realm`` and ``url`` attributes from the
77 configuration.
78 """
79 from .util import _load_pypirc
80 cfg = _load_pypirc(self)
81 self.username = cfg.get('username')
82 self.password = cfg.get('password')
83 self.realm = cfg.get('realm', 'pypi')
84 self.url = cfg.get('repository', self.url)
85
86 def save_configuration(self):
87 """
88 Save the PyPI access configuration. You must have set ``username`` and
89 ``password`` attributes before calling this method.
90 """
91 self.check_credentials()
92 from .util import _store_pypirc
93 _store_pypirc(self)
94
95 def check_credentials(self):
96 """
97 Check that ``username`` and ``password`` have been set, and raise an
98 exception if not.
99 """
100 if self.username is None or self.password is None:
101 raise DistlibException('username and password must be set')
102 pm = HTTPPasswordMgr()
103 _, netloc, _, _, _, _ = urlparse(self.url)
104 pm.add_password(self.realm, netloc, self.username, self.password)
105 self.password_handler = HTTPBasicAuthHandler(pm)
106
107 def register(self, metadata): # pragma: no cover
108 """
109 Register a distribution on PyPI, using the provided metadata.
110
111 :param metadata: A :class:`Metadata` instance defining at least a name
112 and version number for the distribution to be
113 registered.
114 :return: The HTTP response received from PyPI upon submission of the
115 request.
116 """
117 self.check_credentials()
118 metadata.validate()
119 d = metadata.todict()
120 d[':action'] = 'verify'
121 request = self.encode_request(d.items(), [])
122 response = self.send_request(request)
123 d[':action'] = 'submit'
124 request = self.encode_request(d.items(), [])
125 return self.send_request(request)
126
127 def _reader(self, name, stream, outbuf):
128 """
129 Thread runner for reading lines of from a subprocess into a buffer.
130
131 :param name: The logical name of the stream (used for logging only).
132 :param stream: The stream to read from. This will typically a pipe
133 connected to the output stream of a subprocess.
134 :param outbuf: The list to append the read lines to.
135 """
136 while True:
137 s = stream.readline()
138 if not s:
139 break
140 s = s.decode('utf-8').rstrip()
141 outbuf.append(s)
142 logger.debug('%s: %s' % (name, s))
143 stream.close()
144
145 def get_sign_command(self, filename, signer, sign_password, keystore=None): # pragma: no cover
146 """
147 Return a suitable command for signing a file.
148
149 :param filename: The pathname to the file to be signed.
150 :param signer: The identifier of the signer of the file.
151 :param sign_password: The passphrase for the signer's
152 private key used for signing.
153 :param keystore: The path to a directory which contains the keys
154 used in verification. If not specified, the
155 instance's ``gpg_home`` attribute is used instead.
156 :return: The signing command as a list suitable to be
157 passed to :class:`subprocess.Popen`.
158 """
159 cmd = [self.gpg, '--status-fd', '2', '--no-tty']
160 if keystore is None:
161 keystore = self.gpg_home
162 if keystore:
163 cmd.extend(['--homedir', keystore])
164 if sign_password is not None:
165 cmd.extend(['--batch', '--passphrase-fd', '0'])
166 td = tempfile.mkdtemp()
167 sf = os.path.join(td, os.path.basename(filename) + '.asc')
168 cmd.extend(['--detach-sign', '--armor', '--local-user',
169 signer, '--output', sf, filename])
170 logger.debug('invoking: %s', ' '.join(cmd))
171 return cmd, sf
172
173 def run_command(self, cmd, input_data=None):
174 """
175 Run a command in a child process , passing it any input data specified.
176
177 :param cmd: The command to run.
178 :param input_data: If specified, this must be a byte string containing
179 data to be sent to the child process.
180 :return: A tuple consisting of the subprocess' exit code, a list of
181 lines read from the subprocess' ``stdout``, and a list of
182 lines read from the subprocess' ``stderr``.
183 """
184 kwargs = {
185 'stdout': subprocess.PIPE,
186 'stderr': subprocess.PIPE,
187 }
188 if input_data is not None:
189 kwargs['stdin'] = subprocess.PIPE
190 stdout = []
191 stderr = []
192 p = subprocess.Popen(cmd, **kwargs)
193 # We don't use communicate() here because we may need to
194 # get clever with interacting with the command
195 t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))
196 t1.start()
197 t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))
198 t2.start()
199 if input_data is not None:
200 p.stdin.write(input_data)
201 p.stdin.close()
202
203 p.wait()
204 t1.join()
205 t2.join()
206 return p.returncode, stdout, stderr
207
208 def sign_file(self, filename, signer, sign_password, keystore=None): # pragma: no cover
209 """
210 Sign a file.
211
212 :param filename: The pathname to the file to be signed.
213 :param signer: The identifier of the signer of the file.
214 :param sign_password: The passphrase for the signer's
215 private key used for signing.
216 :param keystore: The path to a directory which contains the keys
217 used in signing. If not specified, the instance's
218 ``gpg_home`` attribute is used instead.
219 :return: The absolute pathname of the file where the signature is
220 stored.
221 """
222 cmd, sig_file = self.get_sign_command(filename, signer, sign_password,
223 keystore)
224 rc, stdout, stderr = self.run_command(cmd,
225 sign_password.encode('utf-8'))
226 if rc != 0:
227 raise DistlibException('sign command failed with error '
228 'code %s' % rc)
229 return sig_file
230
231 def upload_file(self, metadata, filename, signer=None, sign_password=None,
232 filetype='sdist', pyversion='source', keystore=None):
233 """
234 Upload a release file to the index.
235
236 :param metadata: A :class:`Metadata` instance defining at least a name
237 and version number for the file to be uploaded.
238 :param filename: The pathname of the file to be uploaded.
239 :param signer: The identifier of the signer of the file.
240 :param sign_password: The passphrase for the signer's
241 private key used for signing.
242 :param filetype: The type of the file being uploaded. This is the
243 distutils command which produced that file, e.g.
244 ``sdist`` or ``bdist_wheel``.
245 :param pyversion: The version of Python which the release relates
246 to. For code compatible with any Python, this would
247 be ``source``, otherwise it would be e.g. ``3.2``.
248 :param keystore: The path to a directory which contains the keys
249 used in signing. If not specified, the instance's
250 ``gpg_home`` attribute is used instead.
251 :return: The HTTP response received from PyPI upon submission of the
252 request.
253 """
254 self.check_credentials()
255 if not os.path.exists(filename):
256 raise DistlibException('not found: %s' % filename)
257 metadata.validate()
258 d = metadata.todict()
259 sig_file = None
260 if signer:
261 if not self.gpg:
262 logger.warning('no signing program available - not signed')
263 else:
264 sig_file = self.sign_file(filename, signer, sign_password,
265 keystore)
266 with open(filename, 'rb') as f:
267 file_data = f.read()
268 md5_digest = hashlib.md5(file_data).hexdigest()
269 sha256_digest = hashlib.sha256(file_data).hexdigest()
270 d.update({
271 ':action': 'file_upload',
272 'protocol_version': '1',
273 'filetype': filetype,
274 'pyversion': pyversion,
275 'md5_digest': md5_digest,
276 'sha256_digest': sha256_digest,
277 })
278 files = [('content', os.path.basename(filename), file_data)]
279 if sig_file:
280 with open(sig_file, 'rb') as f:
281 sig_data = f.read()
282 files.append(('gpg_signature', os.path.basename(sig_file),
283 sig_data))
284 shutil.rmtree(os.path.dirname(sig_file))
285 request = self.encode_request(d.items(), files)
286 return self.send_request(request)
287
288 def upload_documentation(self, metadata, doc_dir): # pragma: no cover
289 """
290 Upload documentation to the index.
291
292 :param metadata: A :class:`Metadata` instance defining at least a name
293 and version number for the documentation to be
294 uploaded.
295 :param doc_dir: The pathname of the directory which contains the
296 documentation. This should be the directory that
297 contains the ``index.html`` for the documentation.
298 :return: The HTTP response received from PyPI upon submission of the
299 request.
300 """
301 self.check_credentials()
302 if not os.path.isdir(doc_dir):
303 raise DistlibException('not a directory: %r' % doc_dir)
304 fn = os.path.join(doc_dir, 'index.html')
305 if not os.path.exists(fn):
306 raise DistlibException('not found: %r' % fn)
307 metadata.validate()
308 name, version = metadata.name, metadata.version
309 zip_data = zip_dir(doc_dir).getvalue()
310 fields = [(':action', 'doc_upload'),
311 ('name', name), ('version', version)]
312 files = [('content', name, zip_data)]
313 request = self.encode_request(fields, files)
314 return self.send_request(request)
315
316 def get_verify_command(self, signature_filename, data_filename,
317 keystore=None):
318 """
319 Return a suitable command for verifying a file.
320
321 :param signature_filename: The pathname to the file containing the
322 signature.
323 :param data_filename: The pathname to the file containing the
324 signed data.
325 :param keystore: The path to a directory which contains the keys
326 used in verification. If not specified, the
327 instance's ``gpg_home`` attribute is used instead.
328 :return: The verifying command as a list suitable to be
329 passed to :class:`subprocess.Popen`.
330 """
331 cmd = [self.gpg, '--status-fd', '2', '--no-tty']
332 if keystore is None:
333 keystore = self.gpg_home
334 if keystore:
335 cmd.extend(['--homedir', keystore])
336 cmd.extend(['--verify', signature_filename, data_filename])
337 logger.debug('invoking: %s', ' '.join(cmd))
338 return cmd
339
340 def verify_signature(self, signature_filename, data_filename,
341 keystore=None):
342 """
343 Verify a signature for a file.
344
345 :param signature_filename: The pathname to the file containing the
346 signature.
347 :param data_filename: The pathname to the file containing the
348 signed data.
349 :param keystore: The path to a directory which contains the keys
350 used in verification. If not specified, the
351 instance's ``gpg_home`` attribute is used instead.
352 :return: True if the signature was verified, else False.
353 """
354 if not self.gpg:
355 raise DistlibException('verification unavailable because gpg '
356 'unavailable')
357 cmd = self.get_verify_command(signature_filename, data_filename,
358 keystore)
359 rc, stdout, stderr = self.run_command(cmd)
360 if rc not in (0, 1):
361 raise DistlibException('verify command failed with error '
362 'code %s' % rc)
363 return rc == 0
364
365 def download_file(self, url, destfile, digest=None, reporthook=None):
366 """
367 This is a convenience method for downloading a file from an URL.
368 Normally, this will be a file from the index, though currently
369 no check is made for this (i.e. a file can be downloaded from
370 anywhere).
371
372 The method is just like the :func:`urlretrieve` function in the
373 standard library, except that it allows digest computation to be
374 done during download and checking that the downloaded data
375 matched any expected value.
376
377 :param url: The URL of the file to be downloaded (assumed to be
378 available via an HTTP GET request).
379 :param destfile: The pathname where the downloaded file is to be
380 saved.
381 :param digest: If specified, this must be a (hasher, value)
382 tuple, where hasher is the algorithm used (e.g.
383 ``'md5'``) and ``value`` is the expected value.
384 :param reporthook: The same as for :func:`urlretrieve` in the
385 standard library.
386 """
387 if digest is None:
388 digester = None
389 logger.debug('No digest specified')
390 else:
391 if isinstance(digest, (list, tuple)):
392 hasher, digest = digest
393 else:
394 hasher = 'md5'
395 digester = getattr(hashlib, hasher)()
396 logger.debug('Digest specified: %s' % digest)
397 # The following code is equivalent to urlretrieve.
398 # We need to do it this way so that we can compute the
399 # digest of the file as we go.
400 with open(destfile, 'wb') as dfp:
401 # addinfourl is not a context manager on 2.x
402 # so we have to use try/finally
403 sfp = self.send_request(Request(url))
404 try:
405 headers = sfp.info()
406 blocksize = 8192
407 size = -1
408 read = 0
409 blocknum = 0
410 if "content-length" in headers:
411 size = int(headers["Content-Length"])
412 if reporthook:
413 reporthook(blocknum, blocksize, size)
414 while True:
415 block = sfp.read(blocksize)
416 if not block:
417 break
418 read += len(block)
419 dfp.write(block)
420 if digester:
421 digester.update(block)
422 blocknum += 1
423 if reporthook:
424 reporthook(blocknum, blocksize, size)
425 finally:
426 sfp.close()
427
428 # check that we got the whole file, if we can
429 if size >= 0 and read < size:
430 raise DistlibException(
431 'retrieval incomplete: got only %d out of %d bytes'
432 % (read, size))
433 # if we have a digest, it must match.
434 if digester:
435 actual = digester.hexdigest()
436 if digest != actual:
437 raise DistlibException('%s digest mismatch for %s: expected '
438 '%s, got %s' % (hasher, destfile,
439 digest, actual))
440 logger.debug('Digest verified: %s', digest)
441
442 def send_request(self, req):
443 """
444 Send a standard library :class:`Request` to PyPI and return its
445 response.
446
447 :param req: The request to send.
448 :return: The HTTP response from PyPI (a standard library HTTPResponse).
449 """
450 handlers = []
451 if self.password_handler:
452 handlers.append(self.password_handler)
453 if self.ssl_verifier:
454 handlers.append(self.ssl_verifier)
455 opener = build_opener(*handlers)
456 return opener.open(req)
457
458 def encode_request(self, fields, files):
459 """
460 Encode fields and files for posting to an HTTP server.
461
462 :param fields: The fields to send as a list of (fieldname, value)
463 tuples.
464 :param files: The files to send as a list of (fieldname, filename,
465 file_bytes) tuple.
466 """
467 # Adapted from packaging, which in turn was adapted from
468 # http://code.activestate.com/recipes/146306
469
470 parts = []
471 boundary = self.boundary
472 for k, values in fields:
473 if not isinstance(values, (list, tuple)):
474 values = [values]
475
476 for v in values:
477 parts.extend((
478 b'--' + boundary,
479 ('Content-Disposition: form-data; name="%s"' %
480 k).encode('utf-8'),
481 b'',
482 v.encode('utf-8')))
483 for key, filename, value in files:
484 parts.extend((
485 b'--' + boundary,
486 ('Content-Disposition: form-data; name="%s"; filename="%s"' %
487 (key, filename)).encode('utf-8'),
488 b'',
489 value))
490
491 parts.extend((b'--' + boundary + b'--', b''))
492
493 body = b'\r\n'.join(parts)
494 ct = b'multipart/form-data; boundary=' + boundary
495 headers = {
496 'Content-type': ct,
497 'Content-length': str(len(body))
498 }
499 return Request(self.url, body, headers)
500
501 def search(self, terms, operator=None): # pragma: no cover
502 if isinstance(terms, string_types):
503 terms = {'name': terms}
504 rpc_proxy = ServerProxy(self.url, timeout=3.0)
505 try:
506 return rpc_proxy.search(terms, operator or 'and')
507 finally:
508 rpc_proxy('close')()
Browse
Usage
Versions