1  #!/usr/bin/env python3.8
       2  
       3  import argparse
       4  import ast
       5  import os
       6  import sys
       7  import time
       8  import traceback
       9  import tokenize
      10  from glob import glob, escape
      11  from pathlib import PurePath
      12  
      13  from typing import List, Optional, Any, Tuple
      14  
      15  sys.path.insert(0, os.getcwd())
      16  from pegen.ast_dump import ast_dump
      17  from pegen.testutil import print_memstats
      18  
      19  SUCCESS = "\033[92m"
      20  FAIL = "\033[91m"
      21  ENDC = "\033[0m"
      22  
      23  COMPILE = 2
      24  PARSE = 1
      25  NOTREE = 0
      26  
      27  argparser = argparse.ArgumentParser(
      28      prog="test_parse_directory",
      29      description="Helper program to test directories or files for pegen",
      30  )
      31  argparser.add_argument("-d", "--directory", help="Directory path containing files to test")
      32  argparser.add_argument(
      33      "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude"
      34  )
      35  argparser.add_argument(
      36      "-s", "--short", action="store_true", help="Only show errors, in a more Emacs-friendly format"
      37  )
      38  argparser.add_argument(
      39      "-v", "--verbose", action="store_true", help="Display detailed errors for failures"
      40  )
      41  
      42  
      43  def report_status(
      44      succeeded: bool,
      45      file: str,
      46      verbose: bool,
      47      error: Optional[Exception] = None,
      48      short: bool = False,
      49  ) -> None:
      50      if short and succeeded:
      51          return
      52  
      53      if succeeded is True:
      54          status = "OK"
      55          COLOR = SUCCESS
      56      else:
      57          status = "Fail"
      58          COLOR = FAIL
      59  
      60      if short:
      61          lineno = 0
      62          offset = 0
      63          if isinstance(error, SyntaxError):
      64              lineno = error.lineno or 1
      65              offset = error.offset or 1
      66              message = error.args[0]
      67          else:
      68              message = f"{error.__class__.__name__}: {error}"
      69          print(f"{file}:{lineno}:{offset}: {message}")
      70      else:
      71          print(f"{COLOR}{file:60} {status}{ENDC}")
      72  
      73          if error and verbose:
      74              print(f"  {str(error.__class__.__name__)}: {error}")
      75  
      76  
      77  def parse_file(source: str, file: str) -> Tuple[Any, float]:
      78      t0 = time.time()
      79      result = ast.parse(source, filename=file)
      80      t1 = time.time()
      81      return result, t1 - t0
      82  
      83  
      84  def generate_time_stats(files, total_seconds) -> None:
      85      total_files = len(files)
      86      total_bytes = 0
      87      total_lines = 0
      88      for file in files:
      89          # Count lines and bytes separately
      90          with open(file, "rb") as f:
      91              total_lines += sum(1 for _ in f)
      92              total_bytes += f.tell()
      93  
      94      print(
      95          f"Checked {total_files:,} files, {total_lines:,} lines,",
      96          f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
      97      )
      98      if total_seconds > 0:
      99          print(
     100              f"That's {total_lines / total_seconds :,.0f} lines/sec,",
     101              f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
     102          )
     103  
     104  
     105  def parse_directory(directory: str, verbose: bool, excluded_files: List[str], short: bool) -> int:
     106      # For a given directory, traverse files and attempt to parse each one
     107      # - Output success/failure for each file
     108      errors = 0
     109      files = []
     110      total_seconds = 0
     111  
     112      for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
     113          # Only attempt to parse Python files and files that are not excluded
     114          if any(PurePath(file).match(pattern) for pattern in excluded_files):
     115              continue
     116  
     117          with tokenize.open(file) as f:
     118              source = f.read()
     119  
     120          try:
     121              result, dt = parse_file(source, file)
     122              total_seconds += dt
     123              report_status(succeeded=True, file=file, verbose=verbose, short=short)
     124          except SyntaxError as error:
     125              report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short)
     126              errors += 1
     127          files.append(file)
     128  
     129      generate_time_stats(files, total_seconds)
     130      if short:
     131          print_memstats()
     132  
     133      if errors:
     134          print(f"Encountered {errors} failures.", file=sys.stderr)
     135          return 1
     136  
     137      return 0
     138  
     139  
     140  def main() -> None:
     141      args = argparser.parse_args()
     142      directory = args.directory
     143      verbose = args.verbose
     144      excluded_files = args.exclude
     145      short = args.short
     146      sys.exit(parse_directory(directory, verbose, excluded_files, short))
     147  
     148  
     149  if __name__ == "__main__":
     150      main()