python (3.12.0)
1 """Configuration management setup
2
3 Some terminology:
4 - name
5 As written in config files.
6 - value
7 Value associated with a name
8 - key
9 Name combined with it's section (section.name)
10 - variant
11 A single word describing where the configuration key-value pair came from
12 """
13
14 import configparser
15 import locale
16 import os
17 import sys
18 from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple
19
20 from pip._internal.exceptions import (
21 ConfigurationError,
22 ConfigurationFileCouldNotBeLoaded,
23 )
24 from pip._internal.utils import appdirs
25 from pip._internal.utils.compat import WINDOWS
26 from pip._internal.utils.logging import getLogger
27 from pip._internal.utils.misc import ensure_dir, enum
28
29 RawConfigParser = configparser.RawConfigParser # Shorthand
30 Kind = NewType("Kind", str)
31
32 CONFIG_BASENAME = "pip.ini" if WINDOWS else "pip.conf"
33 ENV_NAMES_IGNORED = "version", "help"
34
35 # The kinds of configurations there are.
36 kinds = enum(
37 USER="user", # User Specific
38 GLOBAL="global", # System Wide
39 SITE="site", # [Virtual] Environment Specific
40 ENV="env", # from PIP_CONFIG_FILE
41 ENV_VAR="env-var", # from Environment Variables
42 )
43 OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR
44 VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE
45
46 logger = getLogger(__name__)
47
48
49 # NOTE: Maybe use the optionx attribute to normalize keynames.
50 def _normalize_name(name: str) -> str:
51 """Make a name consistent regardless of source (environment or file)"""
52 name = name.lower().replace("_", "-")
53 if name.startswith("--"):
54 name = name[2:] # only prefer long opts
55 return name
56
57
58 def _disassemble_key(name: str) -> List[str]:
59 if "." not in name:
60 error_message = (
61 "Key does not contain dot separated section and key. "
62 "Perhaps you wanted to use 'global.{}' instead?"
63 ).format(name)
64 raise ConfigurationError(error_message)
65 return name.split(".", 1)
66
67
68 def get_configuration_files() -> Dict[Kind, List[str]]:
69 global_config_files = [
70 os.path.join(path, CONFIG_BASENAME) for path in appdirs.site_config_dirs("pip")
71 ]
72
73 site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME)
74 legacy_config_file = os.path.join(
75 os.path.expanduser("~"),
76 "pip" if WINDOWS else ".pip",
77 CONFIG_BASENAME,
78 )
79 new_config_file = os.path.join(appdirs.user_config_dir("pip"), CONFIG_BASENAME)
80 return {
81 kinds.GLOBAL: global_config_files,
82 kinds.SITE: [site_config_file],
83 kinds.USER: [legacy_config_file, new_config_file],
84 }
85
86
87 class ESC[4;38;5;81mConfiguration:
88 """Handles management of configuration.
89
90 Provides an interface to accessing and managing configuration files.
91
92 This class converts provides an API that takes "section.key-name" style
93 keys and stores the value associated with it as "key-name" under the
94 section "section".
95
96 This allows for a clean interface wherein the both the section and the
97 key-name are preserved in an easy to manage form in the configuration files
98 and the data stored is also nice.
99 """
100
101 def __init__(self, isolated: bool, load_only: Optional[Kind] = None) -> None:
102 super().__init__()
103
104 if load_only is not None and load_only not in VALID_LOAD_ONLY:
105 raise ConfigurationError(
106 "Got invalid value for load_only - should be one of {}".format(
107 ", ".join(map(repr, VALID_LOAD_ONLY))
108 )
109 )
110 self.isolated = isolated
111 self.load_only = load_only
112
113 # Because we keep track of where we got the data from
114 self._parsers: Dict[Kind, List[Tuple[str, RawConfigParser]]] = {
115 variant: [] for variant in OVERRIDE_ORDER
116 }
117 self._config: Dict[Kind, Dict[str, Any]] = {
118 variant: {} for variant in OVERRIDE_ORDER
119 }
120 self._modified_parsers: List[Tuple[str, RawConfigParser]] = []
121
122 def load(self) -> None:
123 """Loads configuration from configuration files and environment"""
124 self._load_config_files()
125 if not self.isolated:
126 self._load_environment_vars()
127
128 def get_file_to_edit(self) -> Optional[str]:
129 """Returns the file with highest priority in configuration"""
130 assert self.load_only is not None, "Need to be specified a file to be editing"
131
132 try:
133 return self._get_parser_to_modify()[0]
134 except IndexError:
135 return None
136
137 def items(self) -> Iterable[Tuple[str, Any]]:
138 """Returns key-value pairs like dict.items() representing the loaded
139 configuration
140 """
141 return self._dictionary.items()
142
143 def get_value(self, key: str) -> Any:
144 """Get a value from the configuration."""
145 orig_key = key
146 key = _normalize_name(key)
147 try:
148 return self._dictionary[key]
149 except KeyError:
150 # disassembling triggers a more useful error message than simply
151 # "No such key" in the case that the key isn't in the form command.option
152 _disassemble_key(key)
153 raise ConfigurationError(f"No such key - {orig_key}")
154
155 def set_value(self, key: str, value: Any) -> None:
156 """Modify a value in the configuration."""
157 key = _normalize_name(key)
158 self._ensure_have_load_only()
159
160 assert self.load_only
161 fname, parser = self._get_parser_to_modify()
162
163 if parser is not None:
164 section, name = _disassemble_key(key)
165
166 # Modify the parser and the configuration
167 if not parser.has_section(section):
168 parser.add_section(section)
169 parser.set(section, name, value)
170
171 self._config[self.load_only][key] = value
172 self._mark_as_modified(fname, parser)
173
174 def unset_value(self, key: str) -> None:
175 """Unset a value in the configuration."""
176 orig_key = key
177 key = _normalize_name(key)
178 self._ensure_have_load_only()
179
180 assert self.load_only
181 if key not in self._config[self.load_only]:
182 raise ConfigurationError(f"No such key - {orig_key}")
183
184 fname, parser = self._get_parser_to_modify()
185
186 if parser is not None:
187 section, name = _disassemble_key(key)
188 if not (
189 parser.has_section(section) and parser.remove_option(section, name)
190 ):
191 # The option was not removed.
192 raise ConfigurationError(
193 "Fatal Internal error [id=1]. Please report as a bug."
194 )
195
196 # The section may be empty after the option was removed.
197 if not parser.items(section):
198 parser.remove_section(section)
199 self._mark_as_modified(fname, parser)
200
201 del self._config[self.load_only][key]
202
203 def save(self) -> None:
204 """Save the current in-memory state."""
205 self._ensure_have_load_only()
206
207 for fname, parser in self._modified_parsers:
208 logger.info("Writing to %s", fname)
209
210 # Ensure directory exists.
211 ensure_dir(os.path.dirname(fname))
212
213 # Ensure directory's permission(need to be writeable)
214 try:
215 with open(fname, "w") as f:
216 parser.write(f)
217 except OSError as error:
218 raise ConfigurationError(
219 f"An error occurred while writing to the configuration file "
220 f"{fname}: {error}"
221 )
222
223 #
224 # Private routines
225 #
226
227 def _ensure_have_load_only(self) -> None:
228 if self.load_only is None:
229 raise ConfigurationError("Needed a specific file to be modifying.")
230 logger.debug("Will be working with %s variant only", self.load_only)
231
232 @property
233 def _dictionary(self) -> Dict[str, Any]:
234 """A dictionary representing the loaded configuration."""
235 # NOTE: Dictionaries are not populated if not loaded. So, conditionals
236 # are not needed here.
237 retval = {}
238
239 for variant in OVERRIDE_ORDER:
240 retval.update(self._config[variant])
241
242 return retval
243
244 def _load_config_files(self) -> None:
245 """Loads configuration from configuration files"""
246 config_files = dict(self.iter_config_files())
247 if config_files[kinds.ENV][0:1] == [os.devnull]:
248 logger.debug(
249 "Skipping loading configuration files due to "
250 "environment's PIP_CONFIG_FILE being os.devnull"
251 )
252 return
253
254 for variant, files in config_files.items():
255 for fname in files:
256 # If there's specific variant set in `load_only`, load only
257 # that variant, not the others.
258 if self.load_only is not None and variant != self.load_only:
259 logger.debug("Skipping file '%s' (variant: %s)", fname, variant)
260 continue
261
262 parser = self._load_file(variant, fname)
263
264 # Keeping track of the parsers used
265 self._parsers[variant].append((fname, parser))
266
267 def _load_file(self, variant: Kind, fname: str) -> RawConfigParser:
268 logger.verbose("For variant '%s', will try loading '%s'", variant, fname)
269 parser = self._construct_parser(fname)
270
271 for section in parser.sections():
272 items = parser.items(section)
273 self._config[variant].update(self._normalized_keys(section, items))
274
275 return parser
276
277 def _construct_parser(self, fname: str) -> RawConfigParser:
278 parser = configparser.RawConfigParser()
279 # If there is no such file, don't bother reading it but create the
280 # parser anyway, to hold the data.
281 # Doing this is useful when modifying and saving files, where we don't
282 # need to construct a parser.
283 if os.path.exists(fname):
284 locale_encoding = locale.getpreferredencoding(False)
285 try:
286 parser.read(fname, encoding=locale_encoding)
287 except UnicodeDecodeError:
288 # See https://github.com/pypa/pip/issues/4963
289 raise ConfigurationFileCouldNotBeLoaded(
290 reason=f"contains invalid {locale_encoding} characters",
291 fname=fname,
292 )
293 except configparser.Error as error:
294 # See https://github.com/pypa/pip/issues/4893
295 raise ConfigurationFileCouldNotBeLoaded(error=error)
296 return parser
297
298 def _load_environment_vars(self) -> None:
299 """Loads configuration from environment variables"""
300 self._config[kinds.ENV_VAR].update(
301 self._normalized_keys(":env:", self.get_environ_vars())
302 )
303
304 def _normalized_keys(
305 self, section: str, items: Iterable[Tuple[str, Any]]
306 ) -> Dict[str, Any]:
307 """Normalizes items to construct a dictionary with normalized keys.
308
309 This routine is where the names become keys and are made the same
310 regardless of source - configuration files or environment.
311 """
312 normalized = {}
313 for name, val in items:
314 key = section + "." + _normalize_name(name)
315 normalized[key] = val
316 return normalized
317
318 def get_environ_vars(self) -> Iterable[Tuple[str, str]]:
319 """Returns a generator with all environmental vars with prefix PIP_"""
320 for key, val in os.environ.items():
321 if key.startswith("PIP_"):
322 name = key[4:].lower()
323 if name not in ENV_NAMES_IGNORED:
324 yield name, val
325
326 # XXX: This is patched in the tests.
327 def iter_config_files(self) -> Iterable[Tuple[Kind, List[str]]]:
328 """Yields variant and configuration files associated with it.
329
330 This should be treated like items of a dictionary.
331 """
332 # SMELL: Move the conditions out of this function
333
334 # environment variables have the lowest priority
335 config_file = os.environ.get("PIP_CONFIG_FILE", None)
336 if config_file is not None:
337 yield kinds.ENV, [config_file]
338 else:
339 yield kinds.ENV, []
340
341 config_files = get_configuration_files()
342
343 # at the base we have any global configuration
344 yield kinds.GLOBAL, config_files[kinds.GLOBAL]
345
346 # per-user configuration next
347 should_load_user_config = not self.isolated and not (
348 config_file and os.path.exists(config_file)
349 )
350 if should_load_user_config:
351 # The legacy config file is overridden by the new config file
352 yield kinds.USER, config_files[kinds.USER]
353
354 # finally virtualenv configuration first trumping others
355 yield kinds.SITE, config_files[kinds.SITE]
356
357 def get_values_in_config(self, variant: Kind) -> Dict[str, Any]:
358 """Get values present in a config file"""
359 return self._config[variant]
360
361 def _get_parser_to_modify(self) -> Tuple[str, RawConfigParser]:
362 # Determine which parser to modify
363 assert self.load_only
364 parsers = self._parsers[self.load_only]
365 if not parsers:
366 # This should not happen if everything works correctly.
367 raise ConfigurationError(
368 "Fatal Internal error [id=2]. Please report as a bug."
369 )
370
371 # Use the highest priority parser.
372 return parsers[-1]
373
374 # XXX: This is patched in the tests.
375 def _mark_as_modified(self, fname: str, parser: RawConfigParser) -> None:
376 file_parser_tuple = (fname, parser)
377 if file_parser_tuple not in self._modified_parsers:
378 self._modified_parsers.append(file_parser_tuple)
379
380 def __repr__(self) -> str:
381 return f"{self.__class__.__name__}({self._dictionary!r})"
Browse
Usage
Versions