diff --git a/.eggs/README.txt b/.eggs/README.txt new file mode 100644 index 0000000..5d01668 --- /dev/null +++ b/.eggs/README.txt @@ -0,0 +1,6 @@ +This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins. + +This directory caches those eggs to prevent repeated downloads. + +However, it is safe to delete this directory. + diff --git a/.gitignore b/.gitignore index 45a067a..f05ae8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ +# fix_SSLError ----- +.env38/ +.vscode/ +pipreqs_flake8_results.txt +pipreqs_setup_test_results.txt +pipreqs_tox_results.txt +# ------------------ + *.py[cod] # C extensions diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 60e73a2..01582b4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -66,7 +66,7 @@ Ready to contribute? Here's how to set up `pipreqs` for local development. $ mkvirtualenv pipreqs $ cd pipreqs/ - $ python setup.py develop + $ python setup.py develop (or $pip install -e .) 4. Create a branch for local development:: @@ -80,7 +80,12 @@ Ready to contribute? Here's how to set up `pipreqs` for local development. $ python setup.py test $ tox - To get flake8 and tox, just pip install them into your virtualenv. + To get flake8 and tox, just pip install them into your virtualenv. (or $pip install -r requirements-dev.txt) + + You may also need to provide `CA_BUNDLE` as an environment variable or parameter in the `tests/.env.test` file. + + $ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS + $ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS 6. Commit your changes and push your branch to GitHub:: @@ -99,9 +104,9 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check - https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the - tests pass for all supported Python versions. +3. The pull request should work for Python 3.7 to 3.10 (3.11 needs work), + and PyPy3. Check https://travis-ci.org/bndr/pipreqs/pull_requests and + make sure that the tests pass for all supported Python versions. Tips ---- diff --git a/README.rst b/README.rst index 16c477a..e1d1931 100644 --- a/README.rst +++ b/README.rst @@ -44,6 +44,12 @@ Usage environments parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" + --verify Use verify to provide a CA_BUNDLE file or directory + with certificates of trusted CAs + You can also just set the environment variable in + your terminal: (`export` for nix and `set` for win) + $ export CA_BUNDLE="/certs/path/certificates.pem" + $ set CA_BUNDLE="C:/certs/path/certificates.pem" --debug Print debug information --ignore ... Ignore extra directories, each separated by a comma --no-follow-links Do not follow symbolic links in the project diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 0ceb402..285a523 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -18,6 +18,12 @@ Options: parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" + --verify Use verify to provide a CA_BUNDLE file or directory + with certificates of trusted CAs + You can also just set the environment variable in + your terminal: (`export` for nix and `set` for win) + $ export CA_BUNDLE="/certs/path/certificates.pem" #or + $ set CA_BUNDLE="C:/certs/path/certificates.pem" --debug Print debug information --ignore ... Ignore extra directories, each separated by a comma --no-follow-links Do not follow symbolic links in the project @@ -36,15 +42,16 @@ Options: | e.g. Flask>=1.1.2 | e.g. Flask """ -from contextlib import contextmanager -import os -import sys -import re -import logging import ast +import logging +import os +import re +import sys import traceback -from docopt import docopt +from contextlib import contextmanager + import requests +from docopt import docopt from yarg import json2package from yarg.exceptions import HTTPError @@ -55,6 +62,7 @@ REGEXP = [ re.compile(r'^from ((?!\.+).*?) import (?:.*)$') ] +CA_BUNDLE = os.environ.get("CA_BUNDLE") @contextmanager def _open(filename=None, mode='r'): @@ -171,13 +179,21 @@ def output_requirements(imports, symbol): def get_imports_info( - imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): + imports, + pypi_server="https://pypi.python.org/pypi/", + proxy=None, + verify=CA_BUNDLE, + c=None, +): result = [] for item in imports: try: response = requests.get( - "{0}{1}/json".format(pypi_server, item), proxies=proxy) + "{0}{1}/json".format(pypi_server, item), + proxies=proxy, + verify=verify, + ) if response.status_code == 200: if hasattr(response.content, 'decode'): data = json2package(response.content.decode()) @@ -429,6 +445,7 @@ def init(args): candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" + verify = None proxy = None if args["--pypi-server"]: pypi_server = args["--pypi-server"] @@ -436,6 +453,9 @@ def init(args): if args["--proxy"]: proxy = {'http': args["--proxy"], 'https': args["--proxy"]} + if args["--verify"]: + verify = args["--verify"] + if args["--use-local"]: logging.debug( "Getting package information ONLY from local installation.") @@ -448,6 +468,7 @@ def init(args): if x.lower() not in [z['name'].lower() for z in local]] imports = local + get_imports_info(difference, proxy=proxy, + verify=verify, pypi_server=pypi_server) # sort imports based on lowercase name of package, similar to `pip freeze`. imports = sorted(imports, key=lambda x: x['name'].lower()) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..322f7d9 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,32 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# > python38 -m venv .env38 +# # activate (.env38) virtual environment + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# upgrade pip +# (.env38)> python.exe -m pip install --upgrade pip + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# development packages +# (.env38)> pip install -r requirements-dev.txt + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# development mode installation of `pipreqs` +# (.env38)> pip install -e . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# fix_SSLError tests.settings (optional) use of .env file +# alternative would be to set CA_BUNDLE environment variable +python-dotenv + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# required per CONTRIBUTING workflow +flake8 +tox # only needed in the environment from which tox is run + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# (optional, personal choice) +# vscode settings: "python.formatting.provider": "black", +#black + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index 959d1b7..3dc71a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ wheel==0.23.0 Yarg==0.1.9 -docopt==0.6.2 \ No newline at end of file +docopt==0.6.2 +requests==2.28.2 diff --git a/setup.py b/setup.py index 8b826ed..f45c23b 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ except ImportError: from pipreqs import __version__ - with open('README.rst') as readme_file: readme = readme_file.read() @@ -15,7 +14,10 @@ with open('HISTORY.rst') as history_file: history = history_file.read().replace('.. :changelog:', '') requirements = [ - 'docopt', 'yarg' + 'docopt', 'yarg', 'requests' +] +tests_requirements = [ + 'python-dotenv', 'flake8' ] setup( @@ -49,6 +51,7 @@ setup( 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', ], + tests_require=tests_requirements, test_suite='tests', entry_points={ 'console_scripts': [ diff --git a/tests/.env.test.example b/tests/.env.test.example new file mode 100644 index 0000000..5e9f357 --- /dev/null +++ b/tests/.env.test.example @@ -0,0 +1,8 @@ +# create a new file named `.env.test` +# and assign CA_BUNDLE to your system path\ca.pem file + +CA_BUNDLE=C:\your\path\and\certificates.pem + +# alternatively you can set this value as an environment variable +# $ set CA_BUNDLE="C:\your\path\and\certificates.pem" # for win OS +# $ export CA_BUNDLE="C:\your\path\and\certificates.pem" # for nix OS diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..b3d1e40 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,3 @@ +# .env.test is system specific +# do not check in to repository +.env.test \ No newline at end of file diff --git a/tests/_data_ignore/requirements.txt b/tests/_data_ignore/requirements.txt new file mode 100644 index 0000000..b7f100d --- /dev/null +++ b/tests/_data_ignore/requirements.txt @@ -0,0 +1,12 @@ +asposestorage==1.0.2 +beautifulsoup4==4.11.1 +boto==2.49.0 +docopt==0.6.2 +Flask==2.2.2 +ipython==8.8.0 +nose==1.3.7 +peewee==3.15.4 +pyflakes==3.0.1 +requests==2.28.2 +SQLAlchemy==1.4.46 +ujson==5.7.0 diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..c870b64 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,28 @@ +""" +Environment variables can be used as a first choice +$ set CA_BUNDLE="certificates.pem" # for win OS +$ export CA_BUNDLE="certificates.pem" # for nix OS + +If environment variables are not found then a second attempt +will be made by loading the values from a .env.test file in +the same directory + +See ./env.test.example for details. +""" + +import importlib +import os + +CA_BUNDLE = os.environ.get("CA_BUNDLE") + +if CA_BUNDLE is None and importlib.find_loader("dotenv"): + # optional loading of values from .env.test file + from pathlib import Path + + import dotenv + + env_test_path = Path(os.path.dirname(__file__) + "/.env.test") + config = dotenv.dotenv_values(env_test_path) + + if config is not None: + CA_BUNDLE = config["CA_BUNDLE"] diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index f82d3db..6f99696 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -8,12 +8,20 @@ test_pipreqs Tests for `pipreqs` module. """ -import unittest import os +import unittest + import requests from pipreqs import pipreqs +CA_BUNDLE = os.environ.get("CA_BUNDLE") + +if CA_BUNDLE is None: + #from .settings import CA_BUNDLE + # ImportError: attempted relative import with no known parent package + from tests.settings import CA_BUNDLE + class TestPipreqs(unittest.TestCase): @@ -49,6 +57,7 @@ class TestPipreqs(unittest.TestCase): "requirements2.txt" ) + def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) self.assertEqual(len(imports), 15) @@ -80,7 +89,9 @@ class TestPipreqs(unittest.TestCase): Test to see that the right number of packages were found on PyPI """ imports = pipreqs.get_all_imports(self.project) - with_info = pipreqs.get_imports_info(imports) + with_info = pipreqs.get_imports_info( + imports, proxy=None, verify=CA_BUNDLE + ) # Should contain 10 items without the "nonexistendmodule" and # "after_method_is_valid_even_if_not_pep8" self.assertEqual(len(with_info), 13) @@ -113,9 +124,21 @@ class TestPipreqs(unittest.TestCase): """ Test that all modules we will test upon are in requirements file """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -130,9 +153,21 @@ class TestPipreqs(unittest.TestCase): Test that items listed in requirements.text are the same as locals expected """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": True, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -145,9 +180,20 @@ class TestPipreqs(unittest.TestCase): Test that we can save requirements.txt correctly to a different path """ - pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, - '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": self.alt_requirement_path, + "--use-local": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--print": False, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -163,9 +209,21 @@ class TestPipreqs(unittest.TestCase): """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, - '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--print": False, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -192,25 +250,39 @@ class TestPipreqs(unittest.TestCase): Test that trying to get a custom pypi sever fails correctly """ self.assertRaises( - requests.exceptions.MissingSchema, pipreqs.init, - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': 'nonexistent'} - ) + requests.exceptions.MissingSchema, + pipreqs.init, + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": "nonexistent", + }, + ) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy':None, '--pypi-server':None, - '--ignore':'.ignored_dir,.ignore_second', - '--diff': None, - '--clean': None, - '--mode': None - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--ignore": ".ignored_dir,.ignore_second", + "--diff": None, + "--clean": None, + "--mode": None, + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() @@ -222,13 +294,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=no-pin """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy': None, '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'no-pin' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "no-pin", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() @@ -240,14 +318,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=gt """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'gt' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "gt", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() @@ -261,14 +344,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=compat """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'compat' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "compat", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() @@ -282,18 +370,36 @@ class TestPipreqs(unittest.TestCase): Test --clean parameter """ pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': None, '--proxy': None, - '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": self.requirements_path, + "--mode": "non-pin", + } + ) with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: @@ -305,19 +411,37 @@ class TestPipreqs(unittest.TestCase): """ cleaned_module = 'sqlalchemy' pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 modules_clean = [m for m in self.modules if m != cleaned_module] pipreqs.init( - {'': self.project_clean, '--savepath': None, - '--print': False, '--use-local': None, '--force': None, - '--proxy': None, '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) + { + "": self.project_clean, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": self.requirements_path, + "--mode": "non-pin", + } + ) with open(self.requirements_path, "r") as f: data = f.read().lower() self.assertTrue(cleaned_module not in data) diff --git a/tox.ini b/tox.ini index aea10ec..ceb5265 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,12 @@ [tox] -envlist = py37, py38, py39, pypy3, flake8 +envlist = py37, py38, py39, py310, pypy3, flake8 [gh-actions] python = 3.7: py37 3.8: py38 3.9: py39 + 3.10: py310 pypy-3.7: pypy3 [testenv] @@ -13,4 +14,39 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/pipreqs commands = python setup.py test deps = - -r{toxinidir}/requirements.txt \ No newline at end of file + -r{toxinidir}/requirements.txt + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# tox -e dev311 +# Python 3.11 runtime error +# - py310 and py311 were not tested prior to this submission +# - not related to this submission (but this needs further debug to understand) +# +# ====================================================================== +# FAIL: test_get_use_local_only (tests.test_pipreqs.TestPipreqs.test_get_use_local_only) +# Test without checking PyPI, check to see if names of local +# ---------------------------------------------------------------------- +# Traceback (most recent call last): +# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 127, in test_get_use_local_only +# self.assertTrue(item["name"].lower() in self.local) +# AssertionError: False is not true +# +# ====================================================================== +# FAIL: test_init_local_only (tests.test_pipreqs.TestPipreqs.test_init_local_only) +# Test that items listed in requirements.text are the same +# ---------------------------------------------------------------------- +# Traceback (most recent call last): +# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 182, in test_init_local_only +# self.assertTrue(item[0].lower() in self.local) +# AssertionError: False is not true +# +# ---------------------------------------------------------------------- +[testenv:dev311] +description = development mode environment for debug of py311 +basepython = python3.11 +deps = {[testenv]deps} +# development mode for debuging +usedevelop = True +commands = python setup.py test + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~