github/pipreqs
1
1
Fork 0
mirror of https://github.com/bndr/pipreqs.git synced 2025-06-06 03:25:21 +00:00
Code Issues 3 Packages Projects Releases Wiki Activity

fix_SSLError with tests

Browse Source
This commit is contained in:
c-w-m 2023-01-14 23:57:26 -07:00
parent e5493d922e
commit 4e6091afe9
14 changed files with 381 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.eggs/README.txt Normal file
View File

@ -0,0 +1,6 @@
This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins.
This directory caches those eggs to prevent repeated downloads.
However, it is safe to delete this directory.

8
.gitignore vendored
View File

@ -1,3 +1,11 @@
# fix_SSLError -----
.env38/
.vscode/
pipreqs_flake8_results.txt
pipreqs_setup_test_results.txt
pipreqs_tox_results.txt
# ------------------
*.py[cod]
# C extensions

15
CONTRIBUTING.rst
View File

@ -66,7 +66,7 @@ Ready to contribute? Here's how to set up `pipreqs` for local development.
$ mkvirtualenv pipreqs
$ cd pipreqs/
$ python setup.py develop
$ python setup.py develop (or $pip install -e .)
4. Create a branch for local development::
@ -80,7 +80,12 @@ Ready to contribute? Here's how to set up `pipreqs` for local development.
$ python setup.py test
$ tox
To get flake8 and tox, just pip install them into your virtualenv.
To get flake8 and tox, just pip install them into your virtualenv. (or $pip install -r requirements-dev.txt)
You may also need to provide `CA_BUNDLE` as an environment variable or parameter in the `tests/.env.test` file.
$ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS
$ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS
6. Commit your changes and push your branch to GitHub::
@ -99,9 +104,9 @@ Before you submit a pull request, check that it meets these guidelines:
2. If the pull request adds functionality, the docs should be updated. Put
your new functionality into a function with a docstring, and add the
feature to the list in README.rst.
3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check
https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the
tests pass for all supported Python versions.
3. The pull request should work for Python 3.7 to 3.10 (3.11 needs work),
and PyPy3. Check https://travis-ci.org/bndr/pipreqs/pull_requests and
make sure that the tests pass for all supported Python versions.
Tips
----

6
README.rst
View File

@ -44,6 +44,12 @@ Usage
environments parameter in your terminal:
$ export HTTP_PROXY="http://10.10.1.10:3128"
$ export HTTPS_PROXY="https://10.10.1.10:1080"
--verify <ca_bundle> Use verify to provide a CA_BUNDLE file or directory
with certificates of trusted CAs
You can also just set the environment variable in
your terminal: (`export` for nix and `set` for win)
$ export CA_BUNDLE="/certs/path/certificates.pem"
$ set CA_BUNDLE="C:/certs/path/certificates.pem"
--debug Print debug information
--ignore <dirs>... Ignore extra directories, each separated by a comma
--no-follow-links Do not follow symbolic links in the project

37
pipreqs/pipreqs.py
View File

@ -18,6 +18,12 @@ Options:
parameter in your terminal:
$ export HTTP_PROXY="http://10.10.1.10:3128"
$ export HTTPS_PROXY="https://10.10.1.10:1080"
--verify <ca_bundle> Use verify to provide a CA_BUNDLE file or directory
with certificates of trusted CAs
You can also just set the environment variable in
your terminal: (`export` for nix and `set` for win)
$ export CA_BUNDLE="/certs/path/certificates.pem" #or
$ set CA_BUNDLE="C:/certs/path/certificates.pem"
--debug Print debug information
--ignore <dirs>... Ignore extra directories, each separated by a comma
--no-follow-links Do not follow symbolic links in the project
@ -36,15 +42,16 @@ Options:
<gt> | e.g. Flask>=1.1.2
<no-pin> | e.g. Flask
"""
from contextlib import contextmanager
import os
import sys
import re
import logging
import ast
import logging
import os
import re
import sys
import traceback
from docopt import docopt
from contextlib import contextmanager
import requests
from docopt import docopt
from yarg import json2package
from yarg.exceptions import HTTPError
@ -55,6 +62,7 @@ REGEXP = [
re.compile(r'^from ((?!\.+).*?) import (?:.*)$')
]
CA_BUNDLE = os.environ.get("CA_BUNDLE")
@contextmanager
def _open(filename=None, mode='r'):
@ -171,13 +179,21 @@ def output_requirements(imports, symbol):
def get_imports_info(
imports, pypi_server="https://pypi.python.org/pypi/", proxy=None):
imports,
pypi_server="https://pypi.python.org/pypi/",
proxy=None,
verify=CA_BUNDLE,
c=None,
):
result = []
for item in imports:
try:
response = requests.get(
"{0}{1}/json".format(pypi_server, item), proxies=proxy)
"{0}{1}/json".format(pypi_server, item),
proxies=proxy,
verify=verify,
)
if response.status_code == 200:
if hasattr(response.content, 'decode'):
data = json2package(response.content.decode())
@ -429,6 +445,7 @@ def init(args):
candidates = get_pkg_names(candidates)
logging.debug("Found imports: " + ", ".join(candidates))
pypi_server = "https://pypi.python.org/pypi/"
verify = None
proxy = None
if args["--pypi-server"]:
pypi_server = args["--pypi-server"]
@ -436,6 +453,9 @@ def init(args):
if args["--proxy"]:
proxy = {'http': args["--proxy"], 'https': args["--proxy"]}
if args["--verify"]:
verify = args["--verify"]
if args["--use-local"]:
logging.debug(
"Getting package information ONLY from local installation.")
@ -448,6 +468,7 @@ def init(args):
if x.lower() not in [z['name'].lower() for z in local]]
imports = local + get_imports_info(difference,
proxy=proxy,
verify=verify,
pypi_server=pypi_server)
# sort imports based on lowercase name of package, similar to `pip freeze`.
imports = sorted(imports, key=lambda x: x['name'].lower())

32
requirements-dev.txt Normal file
View File

@ -0,0 +1,32 @@
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# > python38 -m venv .env38
# # activate (.env38) virtual environment
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# upgrade pip
# (.env38)> python.exe -m pip install --upgrade pip
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# development packages
# (.env38)> pip install -r requirements-dev.txt
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# development mode installation of `pipreqs`
# (.env38)> pip install -e .
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# fix_SSLError tests.settings (optional) use of .env file
# alternative would be to set CA_BUNDLE environment variable
python-dotenv
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# required per CONTRIBUTING workflow
flake8
tox # only needed in the environment from which tox is run
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# (optional, personal choice)
# vscode settings: "python.formatting.provider": "black",
#black
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

3
requirements.txt
View File

@ -1,3 +1,4 @@
wheel==0.23.0
Yarg==0.1.9
docopt==0.6.2
docopt==0.6.2
requests==2.28.2

7
setup.py
View File

@ -7,7 +7,6 @@ except ImportError:
from pipreqs import __version__
with open('README.rst') as readme_file:
readme = readme_file.read()
@ -15,7 +14,10 @@ with open('HISTORY.rst') as history_file:
history = history_file.read().replace('.. :changelog:', '')
requirements = [
'docopt', 'yarg'
'docopt', 'yarg', 'requests'
]
tests_requirements = [
'python-dotenv', 'flake8'
]
setup(
@ -49,6 +51,7 @@ setup(
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
],
tests_require=tests_requirements,
test_suite='tests',
entry_points={
'console_scripts': [

8
tests/.env.test.example Normal file
View File

@ -0,0 +1,8 @@
# create a new file named `.env.test`
# and assign CA_BUNDLE to your system path\ca.pem file
CA_BUNDLE=C:\your\path\and\certificates.pem
# alternatively you can set this value as an environment variable
# $ set CA_BUNDLE="C:\your\path\and\certificates.pem" # for win OS
# $ export CA_BUNDLE="C:\your\path\and\certificates.pem" # for nix OS

3
tests/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# .env.test is system specific
# do not check in to repository
.env.test

12
tests/_data_ignore/requirements.txt Normal file
View File

@ -0,0 +1,12 @@
asposestorage==1.0.2
beautifulsoup4==4.11.1
boto==2.49.0
docopt==0.6.2
Flask==2.2.2
ipython==8.8.0
nose==1.3.7
peewee==3.15.4
pyflakes==3.0.1
requests==2.28.2
SQLAlchemy==1.4.46
ujson==5.7.0

28
tests/settings.py Normal file
View File

@ -0,0 +1,28 @@
"""
Environment variables can be used as a first choice
$ set CA_BUNDLE="certificates.pem" # for win OS
$ export CA_BUNDLE="certificates.pem" # for nix OS
If environment variables are not found then a second attempt
will be made by loading the values from a .env.test file in
the same directory
See ./env.test.example for details.
"""
import importlib
import os
CA_BUNDLE = os.environ.get("CA_BUNDLE")
if CA_BUNDLE is None and importlib.find_loader("dotenv"):
# optional loading of values from .env.test file
from pathlib import Path
import dotenv
env_test_path = Path(os.path.dirname(__file__) + "/.env.test")
config = dotenv.dotenv_values(env_test_path)
if config is not None:
CA_BUNDLE = config["CA_BUNDLE"]

264
tests/test_pipreqs.py
View File

@ -8,12 +8,20 @@ test_pipreqs
Tests for `pipreqs` module.
"""
import unittest
import os
import unittest
import requests
from pipreqs import pipreqs
CA_BUNDLE = os.environ.get("CA_BUNDLE")
if CA_BUNDLE is None:
#from .settings import CA_BUNDLE
# ImportError: attempted relative import with no known parent package
from tests.settings import CA_BUNDLE
class TestPipreqs(unittest.TestCase):
@ -49,6 +57,7 @@ class TestPipreqs(unittest.TestCase):
"requirements2.txt"
)
def test_get_all_imports(self):
imports = pipreqs.get_all_imports(self.project)
self.assertEqual(len(imports), 15)
@ -80,7 +89,9 @@ class TestPipreqs(unittest.TestCase):
Test to see that the right number of packages were found on PyPI
"""
imports = pipreqs.get_all_imports(self.project)
with_info = pipreqs.get_imports_info(imports)
with_info = pipreqs.get_imports_info(
imports, proxy=None, verify=CA_BUNDLE
)
# Should contain 10 items without the "nonexistendmodule" and
# "after_method_is_valid_even_if_not_pep8"
self.assertEqual(len(with_info), 13)
@ -113,9 +124,21 @@ class TestPipreqs(unittest.TestCase):
"""
Test that all modules we will test upon are in requirements file
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
@ -130,9 +153,21 @@ class TestPipreqs(unittest.TestCase):
Test that items listed in requirements.text are the same
as locals expected
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": True,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.readlines()
@ -145,9 +180,20 @@ class TestPipreqs(unittest.TestCase):
Test that we can save requirements.txt correctly
to a different path
"""
pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path,
'--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": self.alt_requirement_path,
"--use-local": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--print": False,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.alt_requirement_path) == 1
with open(self.alt_requirement_path, "r") as f:
data = f.read().lower()
@ -163,9 +209,21 @@ class TestPipreqs(unittest.TestCase):
"""
with open(self.requirements_path, "w") as f:
f.write("should_not_be_overwritten")
pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None,
'--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--print": False,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
@ -192,25 +250,39 @@ class TestPipreqs(unittest.TestCase):
Test that trying to get a custom pypi sever fails correctly
"""
self.assertRaises(
requests.exceptions.MissingSchema, pipreqs.init,
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': 'nonexistent'}
)
requests.exceptions.MissingSchema,
pipreqs.init,
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": "nonexistent",
},
)
def test_ignored_directory(self):
"""
Test --ignore parameter
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy':None, '--pypi-server':None,
'--ignore':'.ignored_dir,.ignore_second',
'--diff': None,
'--clean': None,
'--mode': None
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--ignore": ".ignored_dir,.ignore_second",
"--diff": None,
"--clean": None,
"--mode": None,
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.read().lower()
@ -222,13 +294,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=no-pin
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy': None, '--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'no-pin'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "no-pin",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.read().lower()
@ -240,14 +318,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=gt
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy': None,
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'gt'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "gt",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
@ -261,14 +344,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=compat
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy': None,
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'compat'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "compat",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
@ -282,18 +370,36 @@ class TestPipreqs(unittest.TestCase):
Test --clean parameter
"""
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': None, '--proxy': None,
'--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": self.requirements_path,
"--mode": "non-pin",
}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
for item in self.modules[:-3]:
@ -305,19 +411,37 @@ class TestPipreqs(unittest.TestCase):
"""
cleaned_module = 'sqlalchemy'
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
modules_clean = [m for m in self.modules if m != cleaned_module]
pipreqs.init(
{'<path>': self.project_clean, '--savepath': None,
'--print': False, '--use-local': None, '--force': None,
'--proxy': None, '--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
{
"<path>": self.project_clean,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": self.requirements_path,
"--mode": "non-pin",
}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
self.assertTrue(cleaned_module not in data)

40
tox.ini
View File

@ -1,11 +1,12 @@
[tox]
envlist = py37, py38, py39, pypy3, flake8
envlist = py37, py38, py39, py310, pypy3, flake8
[gh-actions]
python =
3.7: py37
3.8: py38
3.9: py39
3.10: py310
pypy-3.7: pypy3
[testenv]
@ -13,4 +14,39 @@ setenv =
PYTHONPATH = {toxinidir}:{toxinidir}/pipreqs
commands = python setup.py test
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/requirements.txt
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# tox -e dev311
# Python 3.11 runtime error
# - py310 and py311 were not tested prior to this submission
# - not related to this submission (but this needs further debug to understand)
#
# ======================================================================
# FAIL: test_get_use_local_only (tests.test_pipreqs.TestPipreqs.test_get_use_local_only)
# Test without checking PyPI, check to see if names of local
# ----------------------------------------------------------------------
# Traceback (most recent call last):
# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 127, in test_get_use_local_only
# self.assertTrue(item["name"].lower() in self.local)
# AssertionError: False is not true
#
# ======================================================================
# FAIL: test_init_local_only (tests.test_pipreqs.TestPipreqs.test_init_local_only)
# Test that items listed in requirements.text are the same
# ----------------------------------------------------------------------
# Traceback (most recent call last):
# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 182, in test_init_local_only
# self.assertTrue(item[0].lower() in self.local)
# AssertionError: False is not true
#
# ----------------------------------------------------------------------
[testenv:dev311]
description = development mode environment for debug of py311
basepython = python3.11
deps = {[testenv]deps}
# development mode for debuging
usedevelop = True
commands = python setup.py test
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~