Merge pull request #473 from ucsd-salad/feat/ignore-errors-flag

Add `--ignore-errors` to skip files with errors

.github/workflows/flake8.yml

@@ -1,18 +1,32 @@
 name: flake8
 
-on: pull_request
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "*"
+    branches:
+      - main
+      - master
+      - develop
+      - "release/*"
+  pull_request:
 
 jobs:
     flake8-lint:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         name: Lint
         steps:
             - name: Check out source repository
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
             - name: Set up Python environment
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
-                  python-version: "3.9"
+                  python-version: "3.13"
             - name: flake8 Lint
               uses: reviewdog/action-flake8@v3
               with:

.github/workflows/tests.yml

@@ -1,46 +1,55 @@
 name: Tests and Codecov
 on:
   push:
+    branches:
+      - master
+      - main
+      - "release/*"
   pull_request:
   workflow_dispatch:
 
 jobs:
     run_tests:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         strategy:
             fail-fast: false
             matrix:
-                python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', 'pypy-3.9-7.3.12']
+                python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.10']
 
         steps:
             - name: Checkout repository
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
 
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
                   python-version: ${{ matrix.python-version }}
 
             - name: Install dependencies
               run: |
-                  python -m pip install --upgrade pip
-                  pip install tox tox-gh-actions
+                  python -m pip install uv
+                  uv pip install --system tox tox-gh-actions
 
             - name: Test with tox
               run: tox
 
     coverage_report:
         needs: run_tests
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         steps:
             - name: Checkout repository
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
+
+            -   name: Set up Python 3.13
+                uses: actions/setup-python@v5
+                with:
+                    python-version: 3.13
 
             - name: Install dependencies
               run: |
-                  python -m pip install --upgrade pip
-                  pip install poetry
-                  poetry install --with dev
+                  python -m pip install uv
+                  uv pip install --system poetry
+                  uv pip install --system .[dev]
 
             - name: Calculate coverage
               run: poetry run coverage run --source=pipreqs -m unittest discover
@@ -49,7 +58,8 @@ jobs:
               run: poetry run coverage xml
 
             - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v3
+              uses: codecov/codecov-action@v5
               with:
                   files: coverage.xml
-                  fail_ci_if_error: true
+                  token: ${{ secrets.CODECOV_TOKEN }}                  
+                  fail_ci_if_error: false

.pre-commit-config.yaml

@@ -0,0 +1,96 @@
+ci:
+    autoupdate_commit_msg: "chore: update pre-commit hooks"
+    autofix_commit_msg: "style: pre-commit fixes"
+    autoupdate_schedule: quarterly
+
+repos:
+    -   repo: https://github.com/pre-commit/pre-commit-hooks
+        rev: v5.0.0
+        hooks:
+            -   id: check-added-large-files
+                args: [ '--maxkb=1000' ]
+            -   id: check-case-conflict
+            -   id: check-merge-conflict
+            -   id: check-symlinks
+            -   id: check-yaml
+            -   id: check-toml
+            -   id: check-json
+            -   id: debug-statements
+            -   id: end-of-file-fixer
+            -   id: mixed-line-ending
+            -   id: requirements-txt-fixer
+            -   id: trailing-whitespace
+                files: ".*\\.(?:tex|py)$"
+                args: [ --markdown-linebreak-ext=md ]
+                exclude: (^notebooks/|^tests/truth/)
+            -   id: detect-private-key
+            -   id: fix-byte-order-marker
+            -   id: check-ast
+            -   id: check-docstring-first
+            -   id: debug-statements
+
+    -   repo: https://github.com/pre-commit/pygrep-hooks
+        rev: v1.10.0
+        hooks:
+            -   id: python-use-type-annotations
+            -   id: python-check-mock-methods
+            -   id: python-no-eval
+            -   id: rst-backticks
+            -   id: rst-directive-colons
+
+    -   repo: https://github.com/asottile/pyupgrade
+        rev: v3.3.1
+        hooks:
+            -   id: pyupgrade
+                args: [ --py38-plus ]
+
+    # Notebook formatting
+    -   repo: https://github.com/nbQA-dev/nbQA
+        rev: 1.9.1
+        hooks:
+            -   id: nbqa-isort
+                additional_dependencies: [ isort ]
+
+            -   id: nbqa-pyupgrade
+                additional_dependencies: [ pyupgrade ]
+                args: [ --py38-plus ]
+
+
+    -   repo: https://github.com/kynan/nbstripout
+        rev: 0.8.1
+        hooks:
+            -   id: nbstripout
+
+    -   repo: https://github.com/sondrelg/pep585-upgrade
+        rev: 'v1.0'
+        hooks:
+            -   id: upgrade-type-hints
+                args: [ '--futures=true' ]
+
+    -   repo: https://github.com/MarcoGorelli/auto-walrus
+        rev: 0.3.4
+        hooks:
+            -   id: auto-walrus
+
+    -   repo: https://github.com/python-jsonschema/check-jsonschema
+        rev: 0.30.0
+        hooks:
+            -   id: check-github-workflows
+            -   id: check-github-actions
+            -   id: check-dependabot
+            -   id: check-readthedocs
+
+    -   repo: https://github.com/dannysepler/rm_unneeded_f_str
+        rev: v0.2.0
+        hooks:
+            -   id: rm-unneeded-f-str
+
+    -   repo: https://github.com/astral-sh/ruff-pre-commit
+        rev: "v0.8.6"
+        hooks:
+            -   id: ruff
+                types_or: [ python, pyi, jupyter ]
+                args: [ --fix, --show-fixes , --line-length=120 ]  # --unsafe-fixes,
+            # Run the formatter.
+            -   id: ruff-format
+                types_or: [ python, pyi, jupyter ]

.python-version

@@ -1,3 +1,4 @@
+3.13
 3.12
 3.11
 3.10

.tool-versions

@@ -1 +1 @@
-python 3.12 3.11 3.10 3.9 3.8 pypy3.9-7.3.12
+python 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.9-7.3.12

README.rst

@@ -2,8 +2,8 @@
 ``pipreqs`` - Generate requirements.txt file for any project based on imports
 =============================================================================
 
-.. image:: https://img.shields.io/travis/bndr/pipreqs.svg
-        :target: https://travis-ci.org/bndr/pipreqs
+.. image:: https://github.com/bndr/pipreqs/actions/workflows/tests.yml/badge.svg
+        :target: https://github.com/bndr/pipreqs/actions/workflows/tests.yml
 
 
 .. image:: https://img.shields.io/pypi/v/pipreqs.svg
@@ -55,6 +55,7 @@ Usage
         --debug               Print debug information
         --ignore <dirs>...    Ignore extra directories, each separated by a comma
         --no-follow-links     Do not follow symbolic links in the project
+        --ignore-errors       Ignore errors while scanning files
         --encoding <charset>  Use encoding parameter for file open
         --savepath <file>     Save the list of requirements in the given file
         --print               Output the list of requirements in the standard output

pipreqs/mapping

@@ -36,6 +36,7 @@ Pyxides:astro_pyxis
 QtCore:PySide
 S3:s3cmd
 SCons:pystick
+speech_recognition:SpeechRecognition
 Shared:Zope2
 Signals:Zope2
 Stemmer:PyStemmer
@@ -582,6 +583,7 @@ ctff:tff
 cups:pycups
 curator:elasticsearch_curator
 curl:pycurl
+cv2:opencv-python
 daemon:python_daemon
 dare:DARE
 dateutil:python_dateutil
@@ -720,6 +722,7 @@ jaraco:jaraco.util
 jinja2:Jinja2
 jiracli:jira_cli
 johnny:johnny_cache
+jose:python_jose
 jpgrid:python_geohash
 jpiarea:python_geohash
 jpype:JPype1
@@ -975,6 +978,7 @@ pysynth_samp:PySynth
 pythongettext:python_gettext
 pythonjsonlogger:python_json_logger
 pyutilib:PyUtilib
+pywintypes:pywin32
 pyximport:Cython
 qs:qserve
 quadtree:python_geohash

pipreqs/pipreqs.py

@@ -20,6 +20,7 @@ Options:
                           $ export HTTPS_PROXY="https://10.10.1.10:1080"
     --debug               Print debug information
     --ignore <dirs>...    Ignore extra directories, each separated by a comma
+    --ignore-errors       Ignore errors while scanning files
     --no-follow-links     Do not follow symbolic links in the project
     --encoding <charset>  Use encoding parameter for file open
     --savepath <file>     Save the list of requirements in the given file
@@ -31,7 +32,7 @@ Options:
     --clean <file>        Clean up requirements.txt by removing modules
                           that are not imported in project
     --mode <scheme>       Enables dynamic versioning with <compat>,
-                          <gt> or <non-pin> schemes.
+                          <gt> or <no-pin> schemes.
                           <compat> | e.g. Flask~=1.1.2
                           <gt>     | e.g. Flask>=1.1.2
                           <no-pin> | e.g. Flask
@@ -97,11 +98,10 @@ def _open(filename=None, mode="r"):
             file.close()
 
 
-def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links=True):
+def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links=True, ignore_errors=False):
     imports = set()
     raw_imports = set()
     candidates = []
-    ignore_errors = False
     ignore_dirs = [
         ".hg",
         ".svn",
@@ -110,6 +110,7 @@ def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links
         "__pycache__",
         "env",
         "venv",
+        ".venv",
         ".ipynb_checkpoints",
     ]
 
@@ -133,9 +134,9 @@ def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links
 
         for file_name in files:
             file_name = os.path.join(root, file_name)
-            contents = read_file_content(file_name, encoding)
 
             try:
+                contents = read_file_content(file_name, encoding)
                 tree = ast.parse(contents)
                 for node in ast.walk(tree):
                     if isinstance(node, ast.Import):
@@ -145,7 +146,7 @@ def get_all_imports(path, encoding="utf-8", extra_ignore_dirs=None, follow_links
                         raw_imports.add(node.module)
             except Exception as exc:
                 if ignore_errors:
-                    traceback.print_exc(exc)
+                    traceback.print_exc()
                     logging.warn("Failed on file: %s" % file_name)
                     continue
                 else:
@@ -504,6 +505,7 @@ def init(args):
     encoding = args.get("--encoding")
     extra_ignore_dirs = args.get("--ignore")
     follow_links = not args.get("--no-follow-links")
+    ignore_errors = args.get("--ignore-errors")
 
     scan_noteboooks = args.get("--scan-notebooks", False)
     handle_scan_noteboooks()
@@ -535,6 +537,7 @@ def init(args):
         encoding=encoding,
         extra_ignore_dirs=extra_ignore_dirs,
         follow_links=follow_links,
+        ignore_errors=ignore_errors,
     )
     candidates = get_pkg_names(candidates)
     logging.debug("Found imports: " + ", ".join(candidates))

pyproject.toml

@@ -1,11 +1,16 @@
-[tool.poetry]
+[project]
 name = "pipreqs"
 version = "0.5.0"
 description = "Pip requirements.txt generator based on imports in project"
-authors = ["Vadim Kravcenko <vadim.kravcenko@gmail.com>"]
+authors = [
+    { name = "Vadim Kravcenko", email = "vadim.kravcenko@gmail.com" }
+]
+maintainers = [
+    {name = "Jonas Eschle", email = "jonas.eschle@gmail.com"}
+]
 license = "Apache-2.0"
-readme = ["README.rst", "HISTORY.rst"]
-packages = [{include = "pipreqs"}]
+readme = "README.rst"
+packages = [{ include = "pipreqs" }]
 repository = "https://github.com/bndr/pipreqs"
 keywords = ["pip", "requirements", "imports"]
 classifiers = [
@@ -14,29 +19,35 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Natural Language :: English",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12"
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
-
-[tool.poetry.scripts]
-pipreqs = "pipreqs.pipreqs:main"
-
-[tool.poetry.dependencies]
-python = ">=3.8.1,<3.13"
-yarg = "0.1.9"
-docopt = "0.6.2"
-nbconvert = "^7.11.0"
-ipython = "8.12.3"
-
-[tool.poetry.group.dev.dependencies]
+requires-python = ">=3.9, <3.14"
+dependencies = [
+    "yarg>=0.1.9",
+    "docopt>=0.6.2",
+    "nbconvert>=7.11.0",
+    "ipython>=8.12.3",
+]
+[project.optional-dependencies]
+dev = [
+    "flake8>=6.1.0",
+    "tox>=4.11.3",
+    "coverage>=7.3.2",
+    "sphinx>=7.2.6;python_version>='3.9'",
+]
+[tool.poetry.group.dev.dependencies]  # for legacy usage
 flake8 = "^6.1.0"
 tox = "^4.11.3"
 coverage = "^7.3.2"
 sphinx = { version = "^7.2.6", python = ">=3.9" }
 
+[project.scripts]
+pipreqs = "pipreqs.pipreqs:main"
+
 [build-system]
-requires = ["poetry-core"]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
 build-backend = "poetry.core.masonry.api"

tests/test_pipreqs.py

@@ -114,6 +114,13 @@ class TestPipreqs(unittest.TestCase):
         """
         self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
 
+    def test_ignore_errors(self):
+        """
+        Test that invalid python files do not raise an exception when ignore_errors is True.
+        """
+        imports = pipreqs.get_all_imports(self.project_invalid, ignore_errors=True)
+        self.assertEqual(len(imports), 0)
+
     def test_get_imports_info(self):
         """
         Test to see that the right number of packages were found on PyPI

tox.ini

@@ -1,20 +1,20 @@
 [tox]
 isolated_build = true
-envlist = py38, py39, py310, py311, py312, pypy3, flake8
+envlist = py39, py310, py311, py312, py313, pypy3, flake8
 
 [gh-actions]
 python =
-    3.8: py38
     3.9: py39
     3.10: py310
     3.11: py311
     3.12: py312
-    pypy-3.9-7.3.12: pypy3
+    3.13: py313
+    pypy-3.10: pypy3
 
 [testenv]
 setenv =
     PYTHONPATH = {toxinidir}:{toxinidir}/pipreqs
-commands = 
+commands =
     python -m unittest discover
 
 [testenv:flake8]
@@ -28,4 +28,4 @@ exclude =
     tests/_data_duplicated_deps/
     tests/_data_ignore/
     tests/_invalid_data/
-max-line-length = 120
+max-line-length = 120