Added Pyre and typeshed d/l

Author: aegilops

CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+## 1.0.2 - 2023-10-16
+
+* Added Pyre
+* Typecheckers now use a current clone of `typeshed` vs their shipped version
+
+## 1.0.1 - 2023-10-10
+
+* Improved error handling
+* Pinning for linter versions
+* Using PyPi location of `flake8-sarif-formatter`
+* Quieter log output
+
 ## 1.0.0 - 2023-10-06
 
 * Initial open source release

action.yml

@@ -8,7 +8,7 @@ inputs:
     description: 'The linter to use'
     required: true
     default: 'flake8'
-    choices: ['ruff', 'flake8', 'pylint', 'mypy', 'pyright', 'pytype', 'fixit']
+    choices: ['ruff', 'flake8', 'pylint', 'mypy', 'pyright', 'pytype', 'fixit', 'pyre']
   target:
     description: 'The target to lint'
     required: true
@@ -50,6 +50,14 @@ inputs:
     description: 'The version of fixit to use'
     required: false
     default: 'latest'
+  pyre-version:
+    description: 'The version of pyre to use'
+    required: false
+    default: 'latest'
+  typeshed-version:
+    description: 'The version of typeshed to use'
+    required: false
+    default: 'main'
 runs:
   using: 'composite'
   steps:
@@ -73,8 +81,9 @@ runs:
         "${PYTHON_CMD}" -mpip install --upgrade pip
 
         # set up linter variables
-        linters=('ruff' 'flake8' 'pylint' 'mypy' 'pyright' 'pytype' 'fixit')
+        linters=('ruff' 'flake8' 'pylint' 'mypy' 'pyright' 'pytype' 'fixit', 'pyre')
         install_flake8_formatter_linters=('ruff', 'flake8')
+        install_typeshed_linters=('pyre', 'pytype', 'mypy', 'pyright')
         EXTRA_PIP_FLAGS=''
         LINTER_VERSION_CONSTRAINT=''
         EXTRA_LINTER_SCRIPT_FLAGS=''
@@ -115,6 +124,10 @@ runs:
               if [[ "${INPUTS_PYTYPE_VERSION}" != "latest" ]]; then
                 LINTER_VERSION_CONSTRAINT="==${INPUTS_PYTYPE_VERSION}"
               fi
+            elif [[ "${INPUTS_LINTER}" == "pyre" ]]; then
+              if [[ "${INPUTS_PYRE_VERSION}" != "latest" ]]; then
+                LINTER_VERSION_CONSTRAINT="==${INPUTS_PYRE_VERSION}"
+              fi
             fi
           fi
 
@@ -141,6 +154,14 @@ runs:
             EXTRA_LINTER_SCRIPT_FLAGS=" --debug"
           fi
 
+          # install typeshed if needed (for typecheckers)
+          if [[ "${install_typeshed_linters[*]}" =~ (^|[^[:alpha:]])${INPUTS_LINTER}([^[:alpha:]]|$) ]]; then
+            echo "::debug::Installing typeshed for ${INPUTS_LINTER}"
+            # clone from GitHub
+            gh repo clone python/typeshed -- --depth 1 --branch "${INPUTS_TYPESHED_VERSION}" "${GITHUB_WORKSPACE}/typeshed" || ( echo "::error::typeshed failed to install for Python ${INPUTS_PYTHON_VERSION}" && exit 1 )
+            EXTRA_LINTER_SCRIPT_FLAGS+=" --typeshed-path=${GITHUB_WORKSPACE}/typeshed"
+          fi
+
           # run linter
           if ! "${PYTHON_CMD}" "${GITHUB_ACTION_PATH}"/python_lint.py "${INPUTS_LINTER}" --target="${INPUTS_TARGET}" --output="${GITHUB_WORKSPACE}/${INPUTS_OUTPUT}" ${EXTRA_LINTER_SCRIPT_FLAGS}; then
             # don't fail "hard" if it's known failures that we cannot account for (yet)
@@ -168,6 +189,8 @@ runs:
         INPUTS_PYRIGHT_VERSION: ${{ inputs.pyright-version }}
         INPUTS_PYTYPE_VERSION: ${{ inputs.pytype-version }}
         INPUTS_FIXIT_VERSION: ${{ inputs.fixit-version }}
+        INPUTS_PYRE_VERSION: ${{ inputs.pyre-version }}
+        INPUTS_TYPESHED_VERSION: ${{ inputs.typeshed-version }}
       shell: bash
     - name: Upload SARIF
       if: ${{ hashFiles(inputs.output) != '' }}

python_lint.py

@@ -11,6 +11,7 @@
 """
 
 import sys
+import os
 import logging
 from argparse import ArgumentParser
 from pathlib import Path
@@ -54,7 +55,7 @@ def make_sarif_run(tool_name: str) -> dict:
     return sarif_run
 
 
-def flake8_linter(target: Path) -> None:
+def flake8_linter(target: Path, *args) -> None:
     """Run the flake8 linter.
 
     In contrast to the other linters, flake8 has plugin architecture.
@@ -154,7 +155,7 @@ def ruff_format_sarif(results: List[Dict[str, Any]], target: Path) -> dict:
     return sarif_run
 
 
-def ruff_linter(target: Path) -> Optional[dict]:
+def ruff_linter(target: Path, *args) -> Optional[dict]:
     """Run the ruff linter."""
     try:
         # pylint: disable=import-outside-toplevel
@@ -256,7 +257,7 @@ def pylint_format_sarif(results: List[Dict[str, Any]], target: Path) -> dict:
     return sarif_run
 
 
-def pylint_linter(target: Path) -> Optional[dict]:
+def pylint_linter(target: Path, *args) -> Optional[dict]:
     """Run the pylint linter."""
     process = run(
         ["pylint", "--output-format=json", "--recursive=y", target.absolute().as_posix()],
@@ -371,7 +372,7 @@ def mypy_format_sarif(mypy_results: str, target: Path) -> dict:
     return sarif_run
 
 
-def mypy_linter(target: Path) -> Optional[dict]:
+def mypy_linter(target: Path, typeshed_path: Path) -> Optional[dict]:
     """Run the mypy linter."""
     mypy_args = [
         "--install-types",
@@ -383,6 +384,8 @@ def mypy_linter(target: Path) -> Optional[dict]:
         "--show-column-numbers",
         "--show-error-end",
         "--show-absolute-path",
+        "--custom-typeshed-dir",
+        typeshed_path.as_posix(),
     ]
 
     process_lint = run(["mypy", *mypy_args, target.absolute().as_posix()], capture_output=True, check=False)
@@ -462,9 +465,9 @@ def pyright_format_sarif(results: dict, target: Path) -> dict:
     return sarif_run
 
 
-def pyright_linter(target: Path) -> Optional[dict]:
+def pyright_linter(target: Path, typeshed_path: Path) -> Optional[dict]:
     """Run the pyright linter."""
-    process = run(["pyright", "--outputjson", target.absolute().as_posix()], capture_output=True, check=False)
+    process = run(["pyright", "--outputjson", "--typeshedpath", typeshed_path, target.absolute().as_posix()], capture_output=True, check=False)
 
     if process.stderr:
         LOG.error("STDERR: %s", process.stderr.decode("utf-8"))
@@ -545,10 +548,12 @@ def pytype_format_sarif(results: str, target: Path) -> dict:
     return sarif_run
 
 
-def pytype_linter(target: Path) -> Optional[dict]:
+def pytype_linter(target: Path, typeshed_path: Path) -> Optional[dict]:
     """Run the pytype linter."""
+    os.environ["TYPESHED_HOME"] = typeshed_path.as_posix()
+
     process = run(
-        ["pytype", "--exclude", ".pytype/", "--", target.absolute().as_posix()], capture_output=True, check=False
+        ["pytype", "--exclude", ".pytype/", "--", target.as_posix()], capture_output=True, check=False,
     )
 
     if process.stderr:
@@ -567,6 +572,33 @@ def pytype_linter(target: Path) -> Optional[dict]:
     return sarif_run
 
 
+def pyre_linter(target: Path, typeshed_path: Path) -> Optional[dict]:
+    """Run the pytype linter."""
+    process = run(
+        ["pyre", "--source-directory", target.as_posix(), "--output", "sarif", "--typeshed", typeshed_path.as_posix(), "check"], capture_output=True, check=False
+    )
+
+    if process.stderr:
+        LOG.debug("STDERR: %s", process.stderr.decode("utf-8"))
+
+    if not process.stdout:
+        LOG.error("No output from pytype")
+        return None
+
+    try:
+        sarif = json.loads(process.stdout.decode("utf-8"))
+    except json.JSONDecodeError as err:
+        LOG.error("Unable to parse pyre output: %s", err)
+        LOG.debug("Output: %s", process.stdout.decode("utf-8"))
+        return None
+
+    if "runs" in sarif and len(sarif["runs"]) > 0:
+        return sarif["runs"][0]
+
+    LOG.error("SARIF not correctly formed, or no runs to output")
+    return None
+
+
 def make_fixit_description(rule: str) -> str:
     """Format 'SomeRuleDescription' into 'Some rule description'."""
     rule = FIND_CAMEL_CASE.sub(lambda x: x.group(0).lower() + " ", rule)
@@ -670,6 +702,7 @@ def make_paths_relative_to_target(runs: List[dict], target: Path) -> None:
     "mypy": mypy_linter,
     "pyright": pyright_linter,
     "fixit": fixit_linter,
+    "pyre": pyre_linter,
 }
 
 # pytype is only supported on Python 3.10 and below, at the time of writing
@@ -682,6 +715,7 @@ def add_args(parser: ArgumentParser) -> None:
     parser.add_argument("linter", choices=LINTERS.keys(), nargs="+", help="The linter(s) to use")
     parser.add_argument("--target", "-t", default=".", required=False, help="Target path for the linter")
     parser.add_argument("--output", "-o", default="python_linter.sarif", required=False, help="Output filename")
+    parser.add_argument("--typeshed-path",required=False, help="Path to typeshed")
     parser.add_argument("--debug", "-d", action="store_true", required=False, help="Enable debug logging")
 
 
@@ -700,11 +734,12 @@ def main() -> None:
     sarif_runs: List[dict] = []
 
     target = Path(args.target).resolve().absolute()
+    typeshed_path = Path(args.typeshed_path).resolve().absolute()
 
     for linter in args.linter:
         LOG.debug("Running %s", linter)
 
-        sarif_run = LINTERS[linter](target)
+        sarif_run = LINTERS[linter](target, typeshed_path)
 
         if sarif_run is not None:
             sarif_runs.append(sarif_run)