From 4e6091afe96065dec36ce0e38cad216f7d4ed1f7 Mon Sep 17 00:00:00 2001 From: c-w-m Date: Sat, 14 Jan 2023 23:57:26 -0700 Subject: [PATCH 1/7] fix_SSLError with tests --- .eggs/README.txt | 6 + .gitignore | 8 + CONTRIBUTING.rst | 15 +- README.rst | 6 + pipreqs/pipreqs.py | 37 +++- requirements-dev.txt | 32 ++++ requirements.txt | 3 +- setup.py | 7 +- tests/.env.test.example | 8 + tests/.gitignore | 3 + tests/_data_ignore/requirements.txt | 12 ++ tests/settings.py | 28 +++ tests/test_pipreqs.py | 264 ++++++++++++++++++++-------- tox.ini | 40 ++++- 14 files changed, 381 insertions(+), 88 deletions(-) create mode 100644 .eggs/README.txt create mode 100644 requirements-dev.txt create mode 100644 tests/.env.test.example create mode 100644 tests/.gitignore create mode 100644 tests/_data_ignore/requirements.txt create mode 100644 tests/settings.py diff --git a/.eggs/README.txt b/.eggs/README.txt new file mode 100644 index 0000000..5d01668 --- /dev/null +++ b/.eggs/README.txt @@ -0,0 +1,6 @@ +This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins. + +This directory caches those eggs to prevent repeated downloads. + +However, it is safe to delete this directory. + diff --git a/.gitignore b/.gitignore index 45a067a..f05ae8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,11 @@ +# fix_SSLError ----- +.env38/ +.vscode/ +pipreqs_flake8_results.txt +pipreqs_setup_test_results.txt +pipreqs_tox_results.txt +# ------------------ + *.py[cod] # C extensions diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 60e73a2..01582b4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -66,7 +66,7 @@ Ready to contribute? Here's how to set up `pipreqs` for local development. $ mkvirtualenv pipreqs $ cd pipreqs/ - $ python setup.py develop + $ python setup.py develop (or $pip install -e .) 4. Create a branch for local development:: @@ -80,7 +80,12 @@ Ready to contribute? Here's how to set up `pipreqs` for local development. $ python setup.py test $ tox - To get flake8 and tox, just pip install them into your virtualenv. + To get flake8 and tox, just pip install them into your virtualenv. (or $pip install -r requirements-dev.txt) + + You may also need to provide `CA_BUNDLE` as an environment variable or parameter in the `tests/.env.test` file. + + $ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS + $ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS 6. Commit your changes and push your branch to GitHub:: @@ -99,9 +104,9 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check - https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the - tests pass for all supported Python versions. +3. The pull request should work for Python 3.7 to 3.10 (3.11 needs work), + and PyPy3. Check https://travis-ci.org/bndr/pipreqs/pull_requests and + make sure that the tests pass for all supported Python versions. Tips ---- diff --git a/README.rst b/README.rst index 16c477a..e1d1931 100644 --- a/README.rst +++ b/README.rst @@ -44,6 +44,12 @@ Usage environments parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" + --verify Use verify to provide a CA_BUNDLE file or directory + with certificates of trusted CAs + You can also just set the environment variable in + your terminal: (`export` for nix and `set` for win) + $ export CA_BUNDLE="/certs/path/certificates.pem" + $ set CA_BUNDLE="C:/certs/path/certificates.pem" --debug Print debug information --ignore ... Ignore extra directories, each separated by a comma --no-follow-links Do not follow symbolic links in the project diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 0ceb402..285a523 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -18,6 +18,12 @@ Options: parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" + --verify Use verify to provide a CA_BUNDLE file or directory + with certificates of trusted CAs + You can also just set the environment variable in + your terminal: (`export` for nix and `set` for win) + $ export CA_BUNDLE="/certs/path/certificates.pem" #or + $ set CA_BUNDLE="C:/certs/path/certificates.pem" --debug Print debug information --ignore ... Ignore extra directories, each separated by a comma --no-follow-links Do not follow symbolic links in the project @@ -36,15 +42,16 @@ Options: | e.g. Flask>=1.1.2 | e.g. Flask """ -from contextlib import contextmanager -import os -import sys -import re -import logging import ast +import logging +import os +import re +import sys import traceback -from docopt import docopt +from contextlib import contextmanager + import requests +from docopt import docopt from yarg import json2package from yarg.exceptions import HTTPError @@ -55,6 +62,7 @@ REGEXP = [ re.compile(r'^from ((?!\.+).*?) import (?:.*)$') ] +CA_BUNDLE = os.environ.get("CA_BUNDLE") @contextmanager def _open(filename=None, mode='r'): @@ -171,13 +179,21 @@ def output_requirements(imports, symbol): def get_imports_info( - imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): + imports, + pypi_server="https://pypi.python.org/pypi/", + proxy=None, + verify=CA_BUNDLE, + c=None, +): result = [] for item in imports: try: response = requests.get( - "{0}{1}/json".format(pypi_server, item), proxies=proxy) + "{0}{1}/json".format(pypi_server, item), + proxies=proxy, + verify=verify, + ) if response.status_code == 200: if hasattr(response.content, 'decode'): data = json2package(response.content.decode()) @@ -429,6 +445,7 @@ def init(args): candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" + verify = None proxy = None if args["--pypi-server"]: pypi_server = args["--pypi-server"] @@ -436,6 +453,9 @@ def init(args): if args["--proxy"]: proxy = {'http': args["--proxy"], 'https': args["--proxy"]} + if args["--verify"]: + verify = args["--verify"] + if args["--use-local"]: logging.debug( "Getting package information ONLY from local installation.") @@ -448,6 +468,7 @@ def init(args): if x.lower() not in [z['name'].lower() for z in local]] imports = local + get_imports_info(difference, proxy=proxy, + verify=verify, pypi_server=pypi_server) # sort imports based on lowercase name of package, similar to `pip freeze`. imports = sorted(imports, key=lambda x: x['name'].lower()) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..322f7d9 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,32 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# > python38 -m venv .env38 +# # activate (.env38) virtual environment + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# upgrade pip +# (.env38)> python.exe -m pip install --upgrade pip + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# development packages +# (.env38)> pip install -r requirements-dev.txt + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# development mode installation of `pipreqs` +# (.env38)> pip install -e . + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# fix_SSLError tests.settings (optional) use of .env file +# alternative would be to set CA_BUNDLE environment variable +python-dotenv + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# required per CONTRIBUTING workflow +flake8 +tox # only needed in the environment from which tox is run + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# (optional, personal choice) +# vscode settings: "python.formatting.provider": "black", +#black + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/requirements.txt b/requirements.txt index 959d1b7..3dc71a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ wheel==0.23.0 Yarg==0.1.9 -docopt==0.6.2 \ No newline at end of file +docopt==0.6.2 +requests==2.28.2 diff --git a/setup.py b/setup.py index 8b826ed..f45c23b 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ except ImportError: from pipreqs import __version__ - with open('README.rst') as readme_file: readme = readme_file.read() @@ -15,7 +14,10 @@ with open('HISTORY.rst') as history_file: history = history_file.read().replace('.. :changelog:', '') requirements = [ - 'docopt', 'yarg' + 'docopt', 'yarg', 'requests' +] +tests_requirements = [ + 'python-dotenv', 'flake8' ] setup( @@ -49,6 +51,7 @@ setup( 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', ], + tests_require=tests_requirements, test_suite='tests', entry_points={ 'console_scripts': [ diff --git a/tests/.env.test.example b/tests/.env.test.example new file mode 100644 index 0000000..5e9f357 --- /dev/null +++ b/tests/.env.test.example @@ -0,0 +1,8 @@ +# create a new file named `.env.test` +# and assign CA_BUNDLE to your system path\ca.pem file + +CA_BUNDLE=C:\your\path\and\certificates.pem + +# alternatively you can set this value as an environment variable +# $ set CA_BUNDLE="C:\your\path\and\certificates.pem" # for win OS +# $ export CA_BUNDLE="C:\your\path\and\certificates.pem" # for nix OS diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..b3d1e40 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,3 @@ +# .env.test is system specific +# do not check in to repository +.env.test \ No newline at end of file diff --git a/tests/_data_ignore/requirements.txt b/tests/_data_ignore/requirements.txt new file mode 100644 index 0000000..b7f100d --- /dev/null +++ b/tests/_data_ignore/requirements.txt @@ -0,0 +1,12 @@ +asposestorage==1.0.2 +beautifulsoup4==4.11.1 +boto==2.49.0 +docopt==0.6.2 +Flask==2.2.2 +ipython==8.8.0 +nose==1.3.7 +peewee==3.15.4 +pyflakes==3.0.1 +requests==2.28.2 +SQLAlchemy==1.4.46 +ujson==5.7.0 diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..c870b64 --- /dev/null +++ b/tests/settings.py @@ -0,0 +1,28 @@ +""" +Environment variables can be used as a first choice +$ set CA_BUNDLE="certificates.pem" # for win OS +$ export CA_BUNDLE="certificates.pem" # for nix OS + +If environment variables are not found then a second attempt +will be made by loading the values from a .env.test file in +the same directory + +See ./env.test.example for details. +""" + +import importlib +import os + +CA_BUNDLE = os.environ.get("CA_BUNDLE") + +if CA_BUNDLE is None and importlib.find_loader("dotenv"): + # optional loading of values from .env.test file + from pathlib import Path + + import dotenv + + env_test_path = Path(os.path.dirname(__file__) + "/.env.test") + config = dotenv.dotenv_values(env_test_path) + + if config is not None: + CA_BUNDLE = config["CA_BUNDLE"] diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index f82d3db..6f99696 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -8,12 +8,20 @@ test_pipreqs Tests for `pipreqs` module. """ -import unittest import os +import unittest + import requests from pipreqs import pipreqs +CA_BUNDLE = os.environ.get("CA_BUNDLE") + +if CA_BUNDLE is None: + #from .settings import CA_BUNDLE + # ImportError: attempted relative import with no known parent package + from tests.settings import CA_BUNDLE + class TestPipreqs(unittest.TestCase): @@ -49,6 +57,7 @@ class TestPipreqs(unittest.TestCase): "requirements2.txt" ) + def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) self.assertEqual(len(imports), 15) @@ -80,7 +89,9 @@ class TestPipreqs(unittest.TestCase): Test to see that the right number of packages were found on PyPI """ imports = pipreqs.get_all_imports(self.project) - with_info = pipreqs.get_imports_info(imports) + with_info = pipreqs.get_imports_info( + imports, proxy=None, verify=CA_BUNDLE + ) # Should contain 10 items without the "nonexistendmodule" and # "after_method_is_valid_even_if_not_pep8" self.assertEqual(len(with_info), 13) @@ -113,9 +124,21 @@ class TestPipreqs(unittest.TestCase): """ Test that all modules we will test upon are in requirements file """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -130,9 +153,21 @@ class TestPipreqs(unittest.TestCase): Test that items listed in requirements.text are the same as locals expected """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": True, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -145,9 +180,20 @@ class TestPipreqs(unittest.TestCase): Test that we can save requirements.txt correctly to a different path """ - pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, - '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": self.alt_requirement_path, + "--use-local": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--print": False, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -163,9 +209,21 @@ class TestPipreqs(unittest.TestCase): """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, - '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init( + { + "": self.project, + "--savepath": None, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--print": False, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -192,25 +250,39 @@ class TestPipreqs(unittest.TestCase): Test that trying to get a custom pypi sever fails correctly """ self.assertRaises( - requests.exceptions.MissingSchema, pipreqs.init, - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': 'nonexistent'} - ) + requests.exceptions.MissingSchema, + pipreqs.init, + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": "nonexistent", + }, + ) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy':None, '--pypi-server':None, - '--ignore':'.ignored_dir,.ignore_second', - '--diff': None, - '--clean': None, - '--mode': None - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--ignore": ".ignored_dir,.ignore_second", + "--diff": None, + "--clean": None, + "--mode": None, + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() @@ -222,13 +294,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=no-pin """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy': None, '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'no-pin' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "no-pin", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() @@ -240,14 +318,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=gt """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'gt' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "gt", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() @@ -261,14 +344,19 @@ class TestPipreqs(unittest.TestCase): Test --mode=compat """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'compat' - } + { + "": self.project_with_ignore_directory, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": "compat", + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() @@ -282,18 +370,36 @@ class TestPipreqs(unittest.TestCase): Test --clean parameter """ pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': None, '--proxy': None, - '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": self.requirements_path, + "--mode": "non-pin", + } + ) with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: @@ -305,19 +411,37 @@ class TestPipreqs(unittest.TestCase): """ cleaned_module = 'sqlalchemy' pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) + { + "": self.project, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": True, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": None, + "--mode": None, + } + ) assert os.path.exists(self.requirements_path) == 1 modules_clean = [m for m in self.modules if m != cleaned_module] pipreqs.init( - {'': self.project_clean, '--savepath': None, - '--print': False, '--use-local': None, '--force': None, - '--proxy': None, '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) + { + "": self.project_clean, + "--savepath": None, + "--print": False, + "--use-local": None, + "--force": None, + "--proxy": None, + "--verify": CA_BUNDLE, + "--pypi-server": None, + "--diff": None, + "--clean": self.requirements_path, + "--mode": "non-pin", + } + ) with open(self.requirements_path, "r") as f: data = f.read().lower() self.assertTrue(cleaned_module not in data) diff --git a/tox.ini b/tox.ini index aea10ec..ceb5265 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,12 @@ [tox] -envlist = py37, py38, py39, pypy3, flake8 +envlist = py37, py38, py39, py310, pypy3, flake8 [gh-actions] python = 3.7: py37 3.8: py38 3.9: py39 + 3.10: py310 pypy-3.7: pypy3 [testenv] @@ -13,4 +14,39 @@ setenv = PYTHONPATH = {toxinidir}:{toxinidir}/pipreqs commands = python setup.py test deps = - -r{toxinidir}/requirements.txt \ No newline at end of file + -r{toxinidir}/requirements.txt + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# tox -e dev311 +# Python 3.11 runtime error +# - py310 and py311 were not tested prior to this submission +# - not related to this submission (but this needs further debug to understand) +# +# ====================================================================== +# FAIL: test_get_use_local_only (tests.test_pipreqs.TestPipreqs.test_get_use_local_only) +# Test without checking PyPI, check to see if names of local +# ---------------------------------------------------------------------- +# Traceback (most recent call last): +# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 127, in test_get_use_local_only +# self.assertTrue(item["name"].lower() in self.local) +# AssertionError: False is not true +# +# ====================================================================== +# FAIL: test_init_local_only (tests.test_pipreqs.TestPipreqs.test_init_local_only) +# Test that items listed in requirements.text are the same +# ---------------------------------------------------------------------- +# Traceback (most recent call last): +# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 182, in test_init_local_only +# self.assertTrue(item[0].lower() in self.local) +# AssertionError: False is not true +# +# ---------------------------------------------------------------------- +[testenv:dev311] +description = development mode environment for debug of py311 +basepython = python3.11 +deps = {[testenv]deps} +# development mode for debuging +usedevelop = True +commands = python setup.py test + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 81b47be64338376bf44255eb3de93cfdee7386d3 Mon Sep 17 00:00:00 2001 From: c-w-m Date: Sun, 15 Jan 2023 00:06:36 -0700 Subject: [PATCH 2/7] clean up comments --- tests/test_pipreqs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index 6f99696..1b3ff5e 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -18,8 +18,6 @@ from pipreqs import pipreqs CA_BUNDLE = os.environ.get("CA_BUNDLE") if CA_BUNDLE is None: - #from .settings import CA_BUNDLE - # ImportError: attempted relative import with no known parent package from tests.settings import CA_BUNDLE From 40a9f181a7ad39eb140489b52f08af400882a7cd Mon Sep 17 00:00:00 2001 From: c-w-m Date: Mon, 16 Jan 2023 07:19:21 -0700 Subject: [PATCH 3/7] sync with branch --- CONTRIBUTING.rst | 1 + requirements-dev.txt | 6 +++++- tests/test_pipreqs.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 01582b4..1c3d1e6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -85,6 +85,7 @@ Ready to contribute? Here's how to set up `pipreqs` for local development. You may also need to provide `CA_BUNDLE` as an environment variable or parameter in the `tests/.env.test` file. $ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS + $ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS 6. Commit your changes and push your branch to GitHub:: diff --git a/requirements-dev.txt b/requirements-dev.txt index 322f7d9..2c8fdbd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,6 +17,9 @@ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # fix_SSLError tests.settings (optional) use of .env file # alternative would be to set CA_BUNDLE environment variable +# $ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS +# $ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS + python-dotenv # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -25,8 +28,9 @@ flake8 tox # only needed in the environment from which tox is run # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# (optional, personal choice) +# (optional) # vscode settings: "python.formatting.provider": "black", #black +docutils # reStructured Text support # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index 1b3ff5e..3b2c49c 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -6,6 +6,11 @@ test_pipreqs ---------------------------------- Tests for `pipreqs` module. + +Environment variables used to mock arguments +e.g., +$ set CA_BUNDLE="certificates.pem" # for win OS +$ export CA_BUNDLE="certificates.pem" # for nix OS """ import os From f43650ca3429235ea02848432f88483decae5b09 Mon Sep 17 00:00:00 2001 From: c-w-m Date: Fri, 20 Jan 2023 15:15:27 -0700 Subject: [PATCH 4/7] str(CA_BUNDLE environment variable) --- pipreqs/pipreqs.py | 6 +++++- requirements-dev.txt | 2 +- tests/.env.test.example | 4 ++-- tests/_data_ignore/requirements.txt | 26 ++++++++++++++------------ tests/settings.py | 5 +++-- tests/test_pipreqs.py | 4 ++++ 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 285a523..1884b85 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -49,6 +49,7 @@ import re import sys import traceback from contextlib import contextmanager +from pathlib import Path import requests from docopt import docopt @@ -63,6 +64,9 @@ REGEXP = [ ] CA_BUNDLE = os.environ.get("CA_BUNDLE") +if CA_BUNDLE is not None: + CA_BUNDLE = str(Path(CA_BUNDLE)) + @contextmanager def _open(filename=None, mode='r'): @@ -445,7 +449,7 @@ def init(args): candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" - verify = None + verify = CA_BUNDLE proxy = None if args["--pypi-server"]: pypi_server = args["--pypi-server"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 2c8fdbd..23cd4a8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ # # activate (.env38) virtual environment # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# upgrade pip +# upgrade pip (21.1.1 -> 22.3.1) # (.env38)> python.exe -m pip install --upgrade pip # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/.env.test.example b/tests/.env.test.example index 5e9f357..1a4a9ed 100644 --- a/tests/.env.test.example +++ b/tests/.env.test.example @@ -4,5 +4,5 @@ CA_BUNDLE=C:\your\path\and\certificates.pem # alternatively you can set this value as an environment variable -# $ set CA_BUNDLE="C:\your\path\and\certificates.pem" # for win OS -# $ export CA_BUNDLE="C:\your\path\and\certificates.pem" # for nix OS +# $ set CA_BUNDLE=C:\your\path\and\certificates.pem # for win OS +# $ export CA_BUNDLE=C:\your\path\and\certificates.pem # for nix OS diff --git a/tests/_data_ignore/requirements.txt b/tests/_data_ignore/requirements.txt index b7f100d..ba40751 100644 --- a/tests/_data_ignore/requirements.txt +++ b/tests/_data_ignore/requirements.txt @@ -1,12 +1,14 @@ -asposestorage==1.0.2 -beautifulsoup4==4.11.1 -boto==2.49.0 -docopt==0.6.2 -Flask==2.2.2 -ipython==8.8.0 -nose==1.3.7 -peewee==3.15.4 -pyflakes==3.0.1 -requests==2.28.2 -SQLAlchemy==1.4.46 -ujson==5.7.0 +asposestorage +beautifulsoup4 +boto +click +docopt +Flask +ipython +nose +pattern +peewee +pyflakes +requests +SQLAlchemy +ujson diff --git a/tests/settings.py b/tests/settings.py index c870b64..68634b4 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -12,13 +12,12 @@ See ./env.test.example for details. import importlib import os +from pathlib import Path CA_BUNDLE = os.environ.get("CA_BUNDLE") if CA_BUNDLE is None and importlib.find_loader("dotenv"): # optional loading of values from .env.test file - from pathlib import Path - import dotenv env_test_path = Path(os.path.dirname(__file__) + "/.env.test") @@ -26,3 +25,5 @@ if CA_BUNDLE is None and importlib.find_loader("dotenv"): if config is not None: CA_BUNDLE = config["CA_BUNDLE"] +elif CA_BUNDLE is not None: + CA_BUNDLE = str(Path(CA_BUNDLE)) diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index 3b2c49c..4e55c7c 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -17,13 +17,17 @@ import os import unittest import requests +from pathlib import Path from pipreqs import pipreqs CA_BUNDLE = os.environ.get("CA_BUNDLE") + if CA_BUNDLE is None: from tests.settings import CA_BUNDLE +else: + CA_BUNDLE = str(Path(CA_BUNDLE)) class TestPipreqs(unittest.TestCase): From 029c38ef8a0331e37c99bcf4586d691b777c1977 Mon Sep 17 00:00:00 2001 From: c-w-m Date: Mon, 23 Jan 2023 14:47:33 -0700 Subject: [PATCH 5/7] alernative approaches to setting CA_BUNDLE --- tests/.env.test.example | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/.env.test.example b/tests/.env.test.example index 1a4a9ed..b3560f8 100644 --- a/tests/.env.test.example +++ b/tests/.env.test.example @@ -3,6 +3,14 @@ CA_BUNDLE=C:\your\path\and\certificates.pem -# alternatively you can set this value as an environment variable +# ---------------------------------------------------------------------------- +# two alternatives to having a .env file +# +# ( 1.) you can set this value as an environment variable # $ set CA_BUNDLE=C:\your\path\and\certificates.pem # for win OS # $ export CA_BUNDLE=C:\your\path\and\certificates.pem # for nix OS +# +# ( 2.) or you can pass the CA_BUNDLE with the `--verify` argument +# example +# > pipreqs --verify C:\your\path\and\certificates.pem . +# ---------------------------------------------------------------------------- From 52c8c044e8eec217b75acfacf166a02657a6f7b8 Mon Sep 17 00:00:00 2001 From: c-w-m Date: Mon, 23 Jan 2023 14:53:29 -0700 Subject: [PATCH 6/7] ignore .env39 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f05ae8c..5263fb3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # fix_SSLError ----- .env38/ +.env39/ .vscode/ pipreqs_flake8_results.txt pipreqs_setup_test_results.txt From cb6b7280bf2da9c10ad46fed58a9b3ca7706aa7d Mon Sep 17 00:00:00 2001 From: c-w-m Date: Mon, 23 Jan 2023 16:13:34 -0700 Subject: [PATCH 7/7] .env39 --- requirements-dev.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 23cd4a8..b4a9ba8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,18 +1,18 @@ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# > python38 -m venv .env38 -# # activate (.env38) virtual environment +# > python39 -m venv .env39 +# # activate (.env39) virtual environment # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # upgrade pip (21.1.1 -> 22.3.1) -# (.env38)> python.exe -m pip install --upgrade pip +# (.env39)> python.exe -m pip install --upgrade pip # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # development packages -# (.env38)> pip install -r requirements-dev.txt +# (.env39)> pip install -r requirements-dev.txt # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # development mode installation of `pipreqs` -# (.env38)> pip install -e . +# (.env39)> pip install -e . # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # fix_SSLError tests.settings (optional) use of .env file