1  #!/usr/bin/env python3.8
       2  
       3  import argparse
       4  import ast
       5  import os
       6  import sys
       7  import time
       8  import tokenize
       9  from glob import glob, escape
      10  from pathlib import PurePath
      11  
      12  from typing import List, Optional, Any, Tuple
      13  
      14  sys.path.insert(0, os.getcwd())
      15  from pegen.testutil import print_memstats
      16  
      17  SUCCESS = "\033[92m"
      18  FAIL = "\033[91m"
      19  ENDC = "\033[0m"
      20  
      21  COMPILE = 2
      22  PARSE = 1
      23  NOTREE = 0
      24  
      25  argparser = argparse.ArgumentParser(
      26      prog="test_parse_directory",
      27      description="Helper program to test directories or files for pegen",
      28  )
      29  argparser.add_argument("-d", "--directory", help="Directory path containing files to test")
      30  argparser.add_argument(
      31      "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude"
      32  )
      33  argparser.add_argument(
      34      "-s", "--short", action="store_true", help="Only show errors, in a more Emacs-friendly format"
      35  )
      36  argparser.add_argument(
      37      "-v", "--verbose", action="store_true", help="Display detailed errors for failures"
      38  )
      39  
      40  
      41  def report_status(
      42      succeeded: bool,
      43      file: str,
      44      verbose: bool,
      45      error: Optional[Exception] = None,
      46      short: bool = False,
      47  ) -> None:
      48      if short and succeeded:
      49          return
      50  
      51      if succeeded is True:
      52          status = "OK"
      53          COLOR = SUCCESS
      54      else:
      55          status = "Fail"
      56          COLOR = FAIL
      57  
      58      if short:
      59          lineno = 0
      60          offset = 0
      61          if isinstance(error, SyntaxError):
      62              lineno = error.lineno or 1
      63              offset = error.offset or 1
      64              message = error.args[0]
      65          else:
      66              message = f"{error.__class__.__name__}: {error}"
      67          print(f"{file}:{lineno}:{offset}: {message}")
      68      else:
      69          print(f"{COLOR}{file:60} {status}{ENDC}")
      70  
      71          if error and verbose:
      72              print(f"  {str(error.__class__.__name__)}: {error}")
      73  
      74  
      75  def parse_file(source: str, file: str) -> Tuple[Any, float]:
      76      t0 = time.time()
      77      result = ast.parse(source, filename=file)
      78      t1 = time.time()
      79      return result, t1 - t0
      80  
      81  
      82  def generate_time_stats(files, total_seconds) -> None:
      83      total_files = len(files)
      84      total_bytes = 0
      85      total_lines = 0
      86      for file in files:
      87          # Count lines and bytes separately
      88          with open(file, "rb") as f:
      89              total_lines += sum(1 for _ in f)
      90              total_bytes += f.tell()
      91  
      92      print(
      93          f"Checked {total_files:,} files, {total_lines:,} lines,",
      94          f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
      95      )
      96      if total_seconds > 0:
      97          print(
      98              f"That's {total_lines / total_seconds :,.0f} lines/sec,",
      99              f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
     100          )
     101  
     102  
     103  def parse_directory(directory: str, verbose: bool, excluded_files: List[str], short: bool) -> int:
     104      # For a given directory, traverse files and attempt to parse each one
     105      # - Output success/failure for each file
     106      errors = 0
     107      files = []
     108      total_seconds = 0
     109  
     110      for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
     111          # Only attempt to parse Python files and files that are not excluded
     112          if any(PurePath(file).match(pattern) for pattern in excluded_files):
     113              continue
     114  
     115          with tokenize.open(file) as f:
     116              source = f.read()
     117  
     118          try:
     119              result, dt = parse_file(source, file)
     120              total_seconds += dt
     121              report_status(succeeded=True, file=file, verbose=verbose, short=short)
     122          except SyntaxError as error:
     123              report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short)
     124              errors += 1
     125          files.append(file)
     126  
     127      generate_time_stats(files, total_seconds)
     128      if short:
     129          print_memstats()
     130  
     131      if errors:
     132          print(f"Encountered {errors} failures.", file=sys.stderr)
     133          return 1
     134  
     135      return 0
     136  
     137  
     138  def main() -> None:
     139      args = argparser.parse_args()
     140      directory = args.directory
     141      verbose = args.verbose
     142      excluded_files = args.exclude
     143      short = args.short
     144      sys.exit(parse_directory(directory, verbose, excluded_files, short))
     145  
     146  
     147  if __name__ == "__main__":
     148      main()