python (3.12.0)
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (C) 2013-2020 Vinay Sajip.
4 # Licensed to the Python Software Foundation under a contributor agreement.
5 # See LICENSE.txt and CONTRIBUTORS.txt.
6 #
7 from __future__ import unicode_literals
8
9 import base64
10 import codecs
11 import datetime
12 from email import message_from_file
13 import hashlib
14 import json
15 import logging
16 import os
17 import posixpath
18 import re
19 import shutil
20 import sys
21 import tempfile
22 import zipfile
23
24 from . import __version__, DistlibException
25 from .compat import sysconfig, ZipFile, fsdecode, text_type, filter
26 from .database import InstalledDistribution
27 from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,
28 LEGACY_METADATA_FILENAME)
29 from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,
30 cached_property, get_cache_base, read_exports, tempdir,
31 get_platform)
32 from .version import NormalizedVersion, UnsupportedVersionError
33
34 logger = logging.getLogger(__name__)
35
36 cache = None # created when needed
37
38 if hasattr(sys, 'pypy_version_info'): # pragma: no cover
39 IMP_PREFIX = 'pp'
40 elif sys.platform.startswith('java'): # pragma: no cover
41 IMP_PREFIX = 'jy'
42 elif sys.platform == 'cli': # pragma: no cover
43 IMP_PREFIX = 'ip'
44 else:
45 IMP_PREFIX = 'cp'
46
47 VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')
48 if not VER_SUFFIX: # pragma: no cover
49 VER_SUFFIX = '%s%s' % sys.version_info[:2]
50 PYVER = 'py' + VER_SUFFIX
51 IMPVER = IMP_PREFIX + VER_SUFFIX
52
53 ARCH = get_platform().replace('-', '_').replace('.', '_')
54
55 ABI = sysconfig.get_config_var('SOABI')
56 if ABI and ABI.startswith('cpython-'):
57 ABI = ABI.replace('cpython-', 'cp').split('-')[0]
58 else:
59 def _derive_abi():
60 parts = ['cp', VER_SUFFIX]
61 if sysconfig.get_config_var('Py_DEBUG'):
62 parts.append('d')
63 if IMP_PREFIX == 'cp':
64 vi = sys.version_info[:2]
65 if vi < (3, 8):
66 wpm = sysconfig.get_config_var('WITH_PYMALLOC')
67 if wpm is None:
68 wpm = True
69 if wpm:
70 parts.append('m')
71 if vi < (3, 3):
72 us = sysconfig.get_config_var('Py_UNICODE_SIZE')
73 if us == 4 or (us is None and sys.maxunicode == 0x10FFFF):
74 parts.append('u')
75 return ''.join(parts)
76 ABI = _derive_abi()
77 del _derive_abi
78
79 FILENAME_RE = re.compile(r'''
80 (?P<nm>[^-]+)
81 -(?P<vn>\d+[^-]*)
82 (-(?P<bn>\d+[^-]*))?
83 -(?P<py>\w+\d+(\.\w+\d+)*)
84 -(?P<bi>\w+)
85 -(?P<ar>\w+(\.\w+)*)
86 \.whl$
87 ''', re.IGNORECASE | re.VERBOSE)
88
89 NAME_VERSION_RE = re.compile(r'''
90 (?P<nm>[^-]+)
91 -(?P<vn>\d+[^-]*)
92 (-(?P<bn>\d+[^-]*))?$
93 ''', re.IGNORECASE | re.VERBOSE)
94
95 SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')
96 SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')
97 SHEBANG_PYTHON = b'#!python'
98 SHEBANG_PYTHONW = b'#!pythonw'
99
100 if os.sep == '/':
101 to_posix = lambda o: o
102 else:
103 to_posix = lambda o: o.replace(os.sep, '/')
104
105 if sys.version_info[0] < 3:
106 import imp
107 else:
108 imp = None
109 import importlib.machinery
110 import importlib.util
111
112 def _get_suffixes():
113 if imp:
114 return [s[0] for s in imp.get_suffixes()]
115 else:
116 return importlib.machinery.EXTENSION_SUFFIXES
117
118 def _load_dynamic(name, path):
119 # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
120 if imp:
121 return imp.load_dynamic(name, path)
122 else:
123 spec = importlib.util.spec_from_file_location(name, path)
124 module = importlib.util.module_from_spec(spec)
125 sys.modules[name] = module
126 spec.loader.exec_module(module)
127 return module
128
129 class ESC[4;38;5;81mMounter(ESC[4;38;5;149mobject):
130 def __init__(self):
131 self.impure_wheels = {}
132 self.libs = {}
133
134 def add(self, pathname, extensions):
135 self.impure_wheels[pathname] = extensions
136 self.libs.update(extensions)
137
138 def remove(self, pathname):
139 extensions = self.impure_wheels.pop(pathname)
140 for k, v in extensions:
141 if k in self.libs:
142 del self.libs[k]
143
144 def find_module(self, fullname, path=None):
145 if fullname in self.libs:
146 result = self
147 else:
148 result = None
149 return result
150
151 def load_module(self, fullname):
152 if fullname in sys.modules:
153 result = sys.modules[fullname]
154 else:
155 if fullname not in self.libs:
156 raise ImportError('unable to find extension for %s' % fullname)
157 result = _load_dynamic(fullname, self.libs[fullname])
158 result.__loader__ = self
159 parts = fullname.rsplit('.', 1)
160 if len(parts) > 1:
161 result.__package__ = parts[0]
162 return result
163
164 _hook = Mounter()
165
166
167 class ESC[4;38;5;81mWheel(ESC[4;38;5;149mobject):
168 """
169 Class to build and install from Wheel files (PEP 427).
170 """
171
172 wheel_version = (1, 1)
173 hash_kind = 'sha256'
174
175 def __init__(self, filename=None, sign=False, verify=False):
176 """
177 Initialise an instance using a (valid) filename.
178 """
179 self.sign = sign
180 self.should_verify = verify
181 self.buildver = ''
182 self.pyver = [PYVER]
183 self.abi = ['none']
184 self.arch = ['any']
185 self.dirname = os.getcwd()
186 if filename is None:
187 self.name = 'dummy'
188 self.version = '0.1'
189 self._filename = self.filename
190 else:
191 m = NAME_VERSION_RE.match(filename)
192 if m:
193 info = m.groupdict('')
194 self.name = info['nm']
195 # Reinstate the local version separator
196 self.version = info['vn'].replace('_', '-')
197 self.buildver = info['bn']
198 self._filename = self.filename
199 else:
200 dirname, filename = os.path.split(filename)
201 m = FILENAME_RE.match(filename)
202 if not m:
203 raise DistlibException('Invalid name or '
204 'filename: %r' % filename)
205 if dirname:
206 self.dirname = os.path.abspath(dirname)
207 self._filename = filename
208 info = m.groupdict('')
209 self.name = info['nm']
210 self.version = info['vn']
211 self.buildver = info['bn']
212 self.pyver = info['py'].split('.')
213 self.abi = info['bi'].split('.')
214 self.arch = info['ar'].split('.')
215
216 @property
217 def filename(self):
218 """
219 Build and return a filename from the various components.
220 """
221 if self.buildver:
222 buildver = '-' + self.buildver
223 else:
224 buildver = ''
225 pyver = '.'.join(self.pyver)
226 abi = '.'.join(self.abi)
227 arch = '.'.join(self.arch)
228 # replace - with _ as a local version separator
229 version = self.version.replace('-', '_')
230 return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver,
231 pyver, abi, arch)
232
233 @property
234 def exists(self):
235 path = os.path.join(self.dirname, self.filename)
236 return os.path.isfile(path)
237
238 @property
239 def tags(self):
240 for pyver in self.pyver:
241 for abi in self.abi:
242 for arch in self.arch:
243 yield pyver, abi, arch
244
245 @cached_property
246 def metadata(self):
247 pathname = os.path.join(self.dirname, self.filename)
248 name_ver = '%s-%s' % (self.name, self.version)
249 info_dir = '%s.dist-info' % name_ver
250 wrapper = codecs.getreader('utf-8')
251 with ZipFile(pathname, 'r') as zf:
252 wheel_metadata = self.get_wheel_metadata(zf)
253 wv = wheel_metadata['Wheel-Version'].split('.', 1)
254 file_version = tuple([int(i) for i in wv])
255 # if file_version < (1, 1):
256 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME,
257 # LEGACY_METADATA_FILENAME]
258 # else:
259 # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]
260 fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME]
261 result = None
262 for fn in fns:
263 try:
264 metadata_filename = posixpath.join(info_dir, fn)
265 with zf.open(metadata_filename) as bf:
266 wf = wrapper(bf)
267 result = Metadata(fileobj=wf)
268 if result:
269 break
270 except KeyError:
271 pass
272 if not result:
273 raise ValueError('Invalid wheel, because metadata is '
274 'missing: looked in %s' % ', '.join(fns))
275 return result
276
277 def get_wheel_metadata(self, zf):
278 name_ver = '%s-%s' % (self.name, self.version)
279 info_dir = '%s.dist-info' % name_ver
280 metadata_filename = posixpath.join(info_dir, 'WHEEL')
281 with zf.open(metadata_filename) as bf:
282 wf = codecs.getreader('utf-8')(bf)
283 message = message_from_file(wf)
284 return dict(message)
285
286 @cached_property
287 def info(self):
288 pathname = os.path.join(self.dirname, self.filename)
289 with ZipFile(pathname, 'r') as zf:
290 result = self.get_wheel_metadata(zf)
291 return result
292
293 def process_shebang(self, data):
294 m = SHEBANG_RE.match(data)
295 if m:
296 end = m.end()
297 shebang, data_after_shebang = data[:end], data[end:]
298 # Preserve any arguments after the interpreter
299 if b'pythonw' in shebang.lower():
300 shebang_python = SHEBANG_PYTHONW
301 else:
302 shebang_python = SHEBANG_PYTHON
303 m = SHEBANG_DETAIL_RE.match(shebang)
304 if m:
305 args = b' ' + m.groups()[-1]
306 else:
307 args = b''
308 shebang = shebang_python + args
309 data = shebang + data_after_shebang
310 else:
311 cr = data.find(b'\r')
312 lf = data.find(b'\n')
313 if cr < 0 or cr > lf:
314 term = b'\n'
315 else:
316 if data[cr:cr + 2] == b'\r\n':
317 term = b'\r\n'
318 else:
319 term = b'\r'
320 data = SHEBANG_PYTHON + term + data
321 return data
322
323 def get_hash(self, data, hash_kind=None):
324 if hash_kind is None:
325 hash_kind = self.hash_kind
326 try:
327 hasher = getattr(hashlib, hash_kind)
328 except AttributeError:
329 raise DistlibException('Unsupported hash algorithm: %r' % hash_kind)
330 result = hasher(data).digest()
331 result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')
332 return hash_kind, result
333
334 def write_record(self, records, record_path, archive_record_path):
335 records = list(records) # make a copy, as mutated
336 records.append((archive_record_path, '', ''))
337 with CSVWriter(record_path) as writer:
338 for row in records:
339 writer.writerow(row)
340
341 def write_records(self, info, libdir, archive_paths):
342 records = []
343 distinfo, info_dir = info
344 hasher = getattr(hashlib, self.hash_kind)
345 for ap, p in archive_paths:
346 with open(p, 'rb') as f:
347 data = f.read()
348 digest = '%s=%s' % self.get_hash(data)
349 size = os.path.getsize(p)
350 records.append((ap, digest, size))
351
352 p = os.path.join(distinfo, 'RECORD')
353 ap = to_posix(os.path.join(info_dir, 'RECORD'))
354 self.write_record(records, p, ap)
355 archive_paths.append((ap, p))
356
357 def build_zip(self, pathname, archive_paths):
358 with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
359 for ap, p in archive_paths:
360 logger.debug('Wrote %s to %s in wheel', p, ap)
361 zf.write(p, ap)
362
363 def build(self, paths, tags=None, wheel_version=None):
364 """
365 Build a wheel from files in specified paths, and use any specified tags
366 when determining the name of the wheel.
367 """
368 if tags is None:
369 tags = {}
370
371 libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
372 if libkey == 'platlib':
373 is_pure = 'false'
374 default_pyver = [IMPVER]
375 default_abi = [ABI]
376 default_arch = [ARCH]
377 else:
378 is_pure = 'true'
379 default_pyver = [PYVER]
380 default_abi = ['none']
381 default_arch = ['any']
382
383 self.pyver = tags.get('pyver', default_pyver)
384 self.abi = tags.get('abi', default_abi)
385 self.arch = tags.get('arch', default_arch)
386
387 libdir = paths[libkey]
388
389 name_ver = '%s-%s' % (self.name, self.version)
390 data_dir = '%s.data' % name_ver
391 info_dir = '%s.dist-info' % name_ver
392
393 archive_paths = []
394
395 # First, stuff which is not in site-packages
396 for key in ('data', 'headers', 'scripts'):
397 if key not in paths:
398 continue
399 path = paths[key]
400 if os.path.isdir(path):
401 for root, dirs, files in os.walk(path):
402 for fn in files:
403 p = fsdecode(os.path.join(root, fn))
404 rp = os.path.relpath(p, path)
405 ap = to_posix(os.path.join(data_dir, key, rp))
406 archive_paths.append((ap, p))
407 if key == 'scripts' and not p.endswith('.exe'):
408 with open(p, 'rb') as f:
409 data = f.read()
410 data = self.process_shebang(data)
411 with open(p, 'wb') as f:
412 f.write(data)
413
414 # Now, stuff which is in site-packages, other than the
415 # distinfo stuff.
416 path = libdir
417 distinfo = None
418 for root, dirs, files in os.walk(path):
419 if root == path:
420 # At the top level only, save distinfo for later
421 # and skip it for now
422 for i, dn in enumerate(dirs):
423 dn = fsdecode(dn)
424 if dn.endswith('.dist-info'):
425 distinfo = os.path.join(root, dn)
426 del dirs[i]
427 break
428 assert distinfo, '.dist-info directory expected, not found'
429
430 for fn in files:
431 # comment out next suite to leave .pyc files in
432 if fsdecode(fn).endswith(('.pyc', '.pyo')):
433 continue
434 p = os.path.join(root, fn)
435 rp = to_posix(os.path.relpath(p, path))
436 archive_paths.append((rp, p))
437
438 # Now distinfo. Assumed to be flat, i.e. os.listdir is enough.
439 files = os.listdir(distinfo)
440 for fn in files:
441 if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
442 p = fsdecode(os.path.join(distinfo, fn))
443 ap = to_posix(os.path.join(info_dir, fn))
444 archive_paths.append((ap, p))
445
446 wheel_metadata = [
447 'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
448 'Generator: distlib %s' % __version__,
449 'Root-Is-Purelib: %s' % is_pure,
450 ]
451 for pyver, abi, arch in self.tags:
452 wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
453 p = os.path.join(distinfo, 'WHEEL')
454 with open(p, 'w') as f:
455 f.write('\n'.join(wheel_metadata))
456 ap = to_posix(os.path.join(info_dir, 'WHEEL'))
457 archive_paths.append((ap, p))
458
459 # sort the entries by archive path. Not needed by any spec, but it
460 # keeps the archive listing and RECORD tidier than they would otherwise
461 # be. Use the number of path segments to keep directory entries together,
462 # and keep the dist-info stuff at the end.
463 def sorter(t):
464 ap = t[0]
465 n = ap.count('/')
466 if '.dist-info' in ap:
467 n += 10000
468 return (n, ap)
469 archive_paths = sorted(archive_paths, key=sorter)
470
471 # Now, at last, RECORD.
472 # Paths in here are archive paths - nothing else makes sense.
473 self.write_records((distinfo, info_dir), libdir, archive_paths)
474 # Now, ready to build the zip file
475 pathname = os.path.join(self.dirname, self.filename)
476 self.build_zip(pathname, archive_paths)
477 return pathname
478
479 def skip_entry(self, arcname):
480 """
481 Determine whether an archive entry should be skipped when verifying
482 or installing.
483 """
484 # The signature file won't be in RECORD,
485 # and we don't currently don't do anything with it
486 # We also skip directories, as they won't be in RECORD
487 # either. See:
488 #
489 # https://github.com/pypa/wheel/issues/294
490 # https://github.com/pypa/wheel/issues/287
491 # https://github.com/pypa/wheel/pull/289
492 #
493 return arcname.endswith(('/', '/RECORD.jws'))
494
495 def install(self, paths, maker, **kwargs):
496 """
497 Install a wheel to the specified paths. If kwarg ``warner`` is
498 specified, it should be a callable, which will be called with two
499 tuples indicating the wheel version of this software and the wheel
500 version in the file, if there is a discrepancy in the versions.
501 This can be used to issue any warnings to raise any exceptions.
502 If kwarg ``lib_only`` is True, only the purelib/platlib files are
503 installed, and the headers, scripts, data and dist-info metadata are
504 not written. If kwarg ``bytecode_hashed_invalidation`` is True, written
505 bytecode will try to use file-hash based invalidation (PEP-552) on
506 supported interpreter versions (CPython 2.7+).
507
508 The return value is a :class:`InstalledDistribution` instance unless
509 ``options.lib_only`` is True, in which case the return value is ``None``.
510 """
511
512 dry_run = maker.dry_run
513 warner = kwargs.get('warner')
514 lib_only = kwargs.get('lib_only', False)
515 bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False)
516
517 pathname = os.path.join(self.dirname, self.filename)
518 name_ver = '%s-%s' % (self.name, self.version)
519 data_dir = '%s.data' % name_ver
520 info_dir = '%s.dist-info' % name_ver
521
522 metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
523 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
524 record_name = posixpath.join(info_dir, 'RECORD')
525
526 wrapper = codecs.getreader('utf-8')
527
528 with ZipFile(pathname, 'r') as zf:
529 with zf.open(wheel_metadata_name) as bwf:
530 wf = wrapper(bwf)
531 message = message_from_file(wf)
532 wv = message['Wheel-Version'].split('.', 1)
533 file_version = tuple([int(i) for i in wv])
534 if (file_version != self.wheel_version) and warner:
535 warner(self.wheel_version, file_version)
536
537 if message['Root-Is-Purelib'] == 'true':
538 libdir = paths['purelib']
539 else:
540 libdir = paths['platlib']
541
542 records = {}
543 with zf.open(record_name) as bf:
544 with CSVReader(stream=bf) as reader:
545 for row in reader:
546 p = row[0]
547 records[p] = row
548
549 data_pfx = posixpath.join(data_dir, '')
550 info_pfx = posixpath.join(info_dir, '')
551 script_pfx = posixpath.join(data_dir, 'scripts', '')
552
553 # make a new instance rather than a copy of maker's,
554 # as we mutate it
555 fileop = FileOperator(dry_run=dry_run)
556 fileop.record = True # so we can rollback if needed
557
558 bc = not sys.dont_write_bytecode # Double negatives. Lovely!
559
560 outfiles = [] # for RECORD writing
561
562 # for script copying/shebang processing
563 workdir = tempfile.mkdtemp()
564 # set target dir later
565 # we default add_launchers to False, as the
566 # Python Launcher should be used instead
567 maker.source_dir = workdir
568 maker.target_dir = None
569 try:
570 for zinfo in zf.infolist():
571 arcname = zinfo.filename
572 if isinstance(arcname, text_type):
573 u_arcname = arcname
574 else:
575 u_arcname = arcname.decode('utf-8')
576 if self.skip_entry(u_arcname):
577 continue
578 row = records[u_arcname]
579 if row[2] and str(zinfo.file_size) != row[2]:
580 raise DistlibException('size mismatch for '
581 '%s' % u_arcname)
582 if row[1]:
583 kind, value = row[1].split('=', 1)
584 with zf.open(arcname) as bf:
585 data = bf.read()
586 _, digest = self.get_hash(data, kind)
587 if digest != value:
588 raise DistlibException('digest mismatch for '
589 '%s' % arcname)
590
591 if lib_only and u_arcname.startswith((info_pfx, data_pfx)):
592 logger.debug('lib_only: skipping %s', u_arcname)
593 continue
594 is_script = (u_arcname.startswith(script_pfx)
595 and not u_arcname.endswith('.exe'))
596
597 if u_arcname.startswith(data_pfx):
598 _, where, rp = u_arcname.split('/', 2)
599 outfile = os.path.join(paths[where], convert_path(rp))
600 else:
601 # meant for site-packages.
602 if u_arcname in (wheel_metadata_name, record_name):
603 continue
604 outfile = os.path.join(libdir, convert_path(u_arcname))
605 if not is_script:
606 with zf.open(arcname) as bf:
607 fileop.copy_stream(bf, outfile)
608 # Issue #147: permission bits aren't preserved. Using
609 # zf.extract(zinfo, libdir) should have worked, but didn't,
610 # see https://www.thetopsites.net/article/53834422.shtml
611 # So ... manually preserve permission bits as given in zinfo
612 if os.name == 'posix':
613 # just set the normal permission bits
614 os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF)
615 outfiles.append(outfile)
616 # Double check the digest of the written file
617 if not dry_run and row[1]:
618 with open(outfile, 'rb') as bf:
619 data = bf.read()
620 _, newdigest = self.get_hash(data, kind)
621 if newdigest != digest:
622 raise DistlibException('digest mismatch '
623 'on write for '
624 '%s' % outfile)
625 if bc and outfile.endswith('.py'):
626 try:
627 pyc = fileop.byte_compile(outfile,
628 hashed_invalidation=bc_hashed_invalidation)
629 outfiles.append(pyc)
630 except Exception:
631 # Don't give up if byte-compilation fails,
632 # but log it and perhaps warn the user
633 logger.warning('Byte-compilation failed',
634 exc_info=True)
635 else:
636 fn = os.path.basename(convert_path(arcname))
637 workname = os.path.join(workdir, fn)
638 with zf.open(arcname) as bf:
639 fileop.copy_stream(bf, workname)
640
641 dn, fn = os.path.split(outfile)
642 maker.target_dir = dn
643 filenames = maker.make(fn)
644 fileop.set_executable_mode(filenames)
645 outfiles.extend(filenames)
646
647 if lib_only:
648 logger.debug('lib_only: returning None')
649 dist = None
650 else:
651 # Generate scripts
652
653 # Try to get pydist.json so we can see if there are
654 # any commands to generate. If this fails (e.g. because
655 # of a legacy wheel), log a warning but don't give up.
656 commands = None
657 file_version = self.info['Wheel-Version']
658 if file_version == '1.0':
659 # Use legacy info
660 ep = posixpath.join(info_dir, 'entry_points.txt')
661 try:
662 with zf.open(ep) as bwf:
663 epdata = read_exports(bwf)
664 commands = {}
665 for key in ('console', 'gui'):
666 k = '%s_scripts' % key
667 if k in epdata:
668 commands['wrap_%s' % key] = d = {}
669 for v in epdata[k].values():
670 s = '%s:%s' % (v.prefix, v.suffix)
671 if v.flags:
672 s += ' [%s]' % ','.join(v.flags)
673 d[v.name] = s
674 except Exception:
675 logger.warning('Unable to read legacy script '
676 'metadata, so cannot generate '
677 'scripts')
678 else:
679 try:
680 with zf.open(metadata_name) as bwf:
681 wf = wrapper(bwf)
682 commands = json.load(wf).get('extensions')
683 if commands:
684 commands = commands.get('python.commands')
685 except Exception:
686 logger.warning('Unable to read JSON metadata, so '
687 'cannot generate scripts')
688 if commands:
689 console_scripts = commands.get('wrap_console', {})
690 gui_scripts = commands.get('wrap_gui', {})
691 if console_scripts or gui_scripts:
692 script_dir = paths.get('scripts', '')
693 if not os.path.isdir(script_dir):
694 raise ValueError('Valid script path not '
695 'specified')
696 maker.target_dir = script_dir
697 for k, v in console_scripts.items():
698 script = '%s = %s' % (k, v)
699 filenames = maker.make(script)
700 fileop.set_executable_mode(filenames)
701
702 if gui_scripts:
703 options = {'gui': True }
704 for k, v in gui_scripts.items():
705 script = '%s = %s' % (k, v)
706 filenames = maker.make(script, options)
707 fileop.set_executable_mode(filenames)
708
709 p = os.path.join(libdir, info_dir)
710 dist = InstalledDistribution(p)
711
712 # Write SHARED
713 paths = dict(paths) # don't change passed in dict
714 del paths['purelib']
715 del paths['platlib']
716 paths['lib'] = libdir
717 p = dist.write_shared_locations(paths, dry_run)
718 if p:
719 outfiles.append(p)
720
721 # Write RECORD
722 dist.write_installed_files(outfiles, paths['prefix'],
723 dry_run)
724 return dist
725 except Exception: # pragma: no cover
726 logger.exception('installation failed.')
727 fileop.rollback()
728 raise
729 finally:
730 shutil.rmtree(workdir)
731
732 def _get_dylib_cache(self):
733 global cache
734 if cache is None:
735 # Use native string to avoid issues on 2.x: see Python #20140.
736 base = os.path.join(get_cache_base(), str('dylib-cache'),
737 '%s.%s' % sys.version_info[:2])
738 cache = Cache(base)
739 return cache
740
741 def _get_extensions(self):
742 pathname = os.path.join(self.dirname, self.filename)
743 name_ver = '%s-%s' % (self.name, self.version)
744 info_dir = '%s.dist-info' % name_ver
745 arcname = posixpath.join(info_dir, 'EXTENSIONS')
746 wrapper = codecs.getreader('utf-8')
747 result = []
748 with ZipFile(pathname, 'r') as zf:
749 try:
750 with zf.open(arcname) as bf:
751 wf = wrapper(bf)
752 extensions = json.load(wf)
753 cache = self._get_dylib_cache()
754 prefix = cache.prefix_to_dir(pathname)
755 cache_base = os.path.join(cache.base, prefix)
756 if not os.path.isdir(cache_base):
757 os.makedirs(cache_base)
758 for name, relpath in extensions.items():
759 dest = os.path.join(cache_base, convert_path(relpath))
760 if not os.path.exists(dest):
761 extract = True
762 else:
763 file_time = os.stat(dest).st_mtime
764 file_time = datetime.datetime.fromtimestamp(file_time)
765 info = zf.getinfo(relpath)
766 wheel_time = datetime.datetime(*info.date_time)
767 extract = wheel_time > file_time
768 if extract:
769 zf.extract(relpath, cache_base)
770 result.append((name, dest))
771 except KeyError:
772 pass
773 return result
774
775 def is_compatible(self):
776 """
777 Determine if a wheel is compatible with the running system.
778 """
779 return is_compatible(self)
780
781 def is_mountable(self):
782 """
783 Determine if a wheel is asserted as mountable by its metadata.
784 """
785 return True # for now - metadata details TBD
786
787 def mount(self, append=False):
788 pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
789 if not self.is_compatible():
790 msg = 'Wheel %s not compatible with this Python.' % pathname
791 raise DistlibException(msg)
792 if not self.is_mountable():
793 msg = 'Wheel %s is marked as not mountable.' % pathname
794 raise DistlibException(msg)
795 if pathname in sys.path:
796 logger.debug('%s already in path', pathname)
797 else:
798 if append:
799 sys.path.append(pathname)
800 else:
801 sys.path.insert(0, pathname)
802 extensions = self._get_extensions()
803 if extensions:
804 if _hook not in sys.meta_path:
805 sys.meta_path.append(_hook)
806 _hook.add(pathname, extensions)
807
808 def unmount(self):
809 pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
810 if pathname not in sys.path:
811 logger.debug('%s not in path', pathname)
812 else:
813 sys.path.remove(pathname)
814 if pathname in _hook.impure_wheels:
815 _hook.remove(pathname)
816 if not _hook.impure_wheels:
817 if _hook in sys.meta_path:
818 sys.meta_path.remove(_hook)
819
820 def verify(self):
821 pathname = os.path.join(self.dirname, self.filename)
822 name_ver = '%s-%s' % (self.name, self.version)
823 data_dir = '%s.data' % name_ver
824 info_dir = '%s.dist-info' % name_ver
825
826 metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
827 wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
828 record_name = posixpath.join(info_dir, 'RECORD')
829
830 wrapper = codecs.getreader('utf-8')
831
832 with ZipFile(pathname, 'r') as zf:
833 with zf.open(wheel_metadata_name) as bwf:
834 wf = wrapper(bwf)
835 message = message_from_file(wf)
836 wv = message['Wheel-Version'].split('.', 1)
837 file_version = tuple([int(i) for i in wv])
838 # TODO version verification
839
840 records = {}
841 with zf.open(record_name) as bf:
842 with CSVReader(stream=bf) as reader:
843 for row in reader:
844 p = row[0]
845 records[p] = row
846
847 for zinfo in zf.infolist():
848 arcname = zinfo.filename
849 if isinstance(arcname, text_type):
850 u_arcname = arcname
851 else:
852 u_arcname = arcname.decode('utf-8')
853 # See issue #115: some wheels have .. in their entries, but
854 # in the filename ... e.g. __main__..py ! So the check is
855 # updated to look for .. in the directory portions
856 p = u_arcname.split('/')
857 if '..' in p:
858 raise DistlibException('invalid entry in '
859 'wheel: %r' % u_arcname)
860
861 if self.skip_entry(u_arcname):
862 continue
863 row = records[u_arcname]
864 if row[2] and str(zinfo.file_size) != row[2]:
865 raise DistlibException('size mismatch for '
866 '%s' % u_arcname)
867 if row[1]:
868 kind, value = row[1].split('=', 1)
869 with zf.open(arcname) as bf:
870 data = bf.read()
871 _, digest = self.get_hash(data, kind)
872 if digest != value:
873 raise DistlibException('digest mismatch for '
874 '%s' % arcname)
875
876 def update(self, modifier, dest_dir=None, **kwargs):
877 """
878 Update the contents of a wheel in a generic way. The modifier should
879 be a callable which expects a dictionary argument: its keys are
880 archive-entry paths, and its values are absolute filesystem paths
881 where the contents the corresponding archive entries can be found. The
882 modifier is free to change the contents of the files pointed to, add
883 new entries and remove entries, before returning. This method will
884 extract the entire contents of the wheel to a temporary location, call
885 the modifier, and then use the passed (and possibly updated)
886 dictionary to write a new wheel. If ``dest_dir`` is specified, the new
887 wheel is written there -- otherwise, the original wheel is overwritten.
888
889 The modifier should return True if it updated the wheel, else False.
890 This method returns the same value the modifier returns.
891 """
892
893 def get_version(path_map, info_dir):
894 version = path = None
895 key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME)
896 if key not in path_map:
897 key = '%s/PKG-INFO' % info_dir
898 if key in path_map:
899 path = path_map[key]
900 version = Metadata(path=path).version
901 return version, path
902
903 def update_version(version, path):
904 updated = None
905 try:
906 v = NormalizedVersion(version)
907 i = version.find('-')
908 if i < 0:
909 updated = '%s+1' % version
910 else:
911 parts = [int(s) for s in version[i + 1:].split('.')]
912 parts[-1] += 1
913 updated = '%s+%s' % (version[:i],
914 '.'.join(str(i) for i in parts))
915 except UnsupportedVersionError:
916 logger.debug('Cannot update non-compliant (PEP-440) '
917 'version %r', version)
918 if updated:
919 md = Metadata(path=path)
920 md.version = updated
921 legacy = path.endswith(LEGACY_METADATA_FILENAME)
922 md.write(path=path, legacy=legacy)
923 logger.debug('Version updated from %r to %r', version,
924 updated)
925
926 pathname = os.path.join(self.dirname, self.filename)
927 name_ver = '%s-%s' % (self.name, self.version)
928 info_dir = '%s.dist-info' % name_ver
929 record_name = posixpath.join(info_dir, 'RECORD')
930 with tempdir() as workdir:
931 with ZipFile(pathname, 'r') as zf:
932 path_map = {}
933 for zinfo in zf.infolist():
934 arcname = zinfo.filename
935 if isinstance(arcname, text_type):
936 u_arcname = arcname
937 else:
938 u_arcname = arcname.decode('utf-8')
939 if u_arcname == record_name:
940 continue
941 if '..' in u_arcname:
942 raise DistlibException('invalid entry in '
943 'wheel: %r' % u_arcname)
944 zf.extract(zinfo, workdir)
945 path = os.path.join(workdir, convert_path(u_arcname))
946 path_map[u_arcname] = path
947
948 # Remember the version.
949 original_version, _ = get_version(path_map, info_dir)
950 # Files extracted. Call the modifier.
951 modified = modifier(path_map, **kwargs)
952 if modified:
953 # Something changed - need to build a new wheel.
954 current_version, path = get_version(path_map, info_dir)
955 if current_version and (current_version == original_version):
956 # Add or update local version to signify changes.
957 update_version(current_version, path)
958 # Decide where the new wheel goes.
959 if dest_dir is None:
960 fd, newpath = tempfile.mkstemp(suffix='.whl',
961 prefix='wheel-update-',
962 dir=workdir)
963 os.close(fd)
964 else:
965 if not os.path.isdir(dest_dir):
966 raise DistlibException('Not a directory: %r' % dest_dir)
967 newpath = os.path.join(dest_dir, self.filename)
968 archive_paths = list(path_map.items())
969 distinfo = os.path.join(workdir, info_dir)
970 info = distinfo, info_dir
971 self.write_records(info, workdir, archive_paths)
972 self.build_zip(newpath, archive_paths)
973 if dest_dir is None:
974 shutil.copyfile(newpath, pathname)
975 return modified
976
977 def _get_glibc_version():
978 import platform
979 ver = platform.libc_ver()
980 result = []
981 if ver[0] == 'glibc':
982 for s in ver[1].split('.'):
983 result.append(int(s) if s.isdigit() else 0)
984 result = tuple(result)
985 return result
986
987 def compatible_tags():
988 """
989 Return (pyver, abi, arch) tuples compatible with this Python.
990 """
991 versions = [VER_SUFFIX]
992 major = VER_SUFFIX[0]
993 for minor in range(sys.version_info[1] - 1, - 1, -1):
994 versions.append(''.join([major, str(minor)]))
995
996 abis = []
997 for suffix in _get_suffixes():
998 if suffix.startswith('.abi'):
999 abis.append(suffix.split('.', 2)[1])
1000 abis.sort()
1001 if ABI != 'none':
1002 abis.insert(0, ABI)
1003 abis.append('none')
1004 result = []
1005
1006 arches = [ARCH]
1007 if sys.platform == 'darwin':
1008 m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)
1009 if m:
1010 name, major, minor, arch = m.groups()
1011 minor = int(minor)
1012 matches = [arch]
1013 if arch in ('i386', 'ppc'):
1014 matches.append('fat')
1015 if arch in ('i386', 'ppc', 'x86_64'):
1016 matches.append('fat3')
1017 if arch in ('ppc64', 'x86_64'):
1018 matches.append('fat64')
1019 if arch in ('i386', 'x86_64'):
1020 matches.append('intel')
1021 if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):
1022 matches.append('universal')
1023 while minor >= 0:
1024 for match in matches:
1025 s = '%s_%s_%s_%s' % (name, major, minor, match)
1026 if s != ARCH: # already there
1027 arches.append(s)
1028 minor -= 1
1029
1030 # Most specific - our Python version, ABI and arch
1031 for abi in abis:
1032 for arch in arches:
1033 result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))
1034 # manylinux
1035 if abi != 'none' and sys.platform.startswith('linux'):
1036 arch = arch.replace('linux_', '')
1037 parts = _get_glibc_version()
1038 if len(parts) == 2:
1039 if parts >= (2, 5):
1040 result.append((''.join((IMP_PREFIX, versions[0])), abi,
1041 'manylinux1_%s' % arch))
1042 if parts >= (2, 12):
1043 result.append((''.join((IMP_PREFIX, versions[0])), abi,
1044 'manylinux2010_%s' % arch))
1045 if parts >= (2, 17):
1046 result.append((''.join((IMP_PREFIX, versions[0])), abi,
1047 'manylinux2014_%s' % arch))
1048 result.append((''.join((IMP_PREFIX, versions[0])), abi,
1049 'manylinux_%s_%s_%s' % (parts[0], parts[1],
1050 arch)))
1051
1052 # where no ABI / arch dependency, but IMP_PREFIX dependency
1053 for i, version in enumerate(versions):
1054 result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))
1055 if i == 0:
1056 result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))
1057
1058 # no IMP_PREFIX, ABI or arch dependency
1059 for i, version in enumerate(versions):
1060 result.append((''.join(('py', version)), 'none', 'any'))
1061 if i == 0:
1062 result.append((''.join(('py', version[0])), 'none', 'any'))
1063
1064 return set(result)
1065
1066
1067 COMPATIBLE_TAGS = compatible_tags()
1068
1069 del compatible_tags
1070
1071
1072 def is_compatible(wheel, tags=None):
1073 if not isinstance(wheel, Wheel):
1074 wheel = Wheel(wheel) # assume it's a filename
1075 result = False
1076 if tags is None:
1077 tags = COMPATIBLE_TAGS
1078 for ver, abi, arch in tags:
1079 if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:
1080 result = True
1081 break
1082 return result
Browse
Usage
Versions