Merge cb6b7280bf2da9c10ad46fed58a9b3ca7706aa7d into 68f9b2859d45a60e699839f87b1b9558cd36a329

This commit is contained in:
c-w-m 2023-10-11 15:38:39 -06:00 committed by GitHub
commit d6ecb1b7b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 409 additions and 88 deletions

6
.eggs/README.txt Normal file
View File

@ -0,0 +1,6 @@
This directory contains eggs that were downloaded by setuptools to build, test, and run plug-ins.
This directory caches those eggs to prevent repeated downloads.
However, it is safe to delete this directory.

9
.gitignore vendored
View File

@ -1,3 +1,12 @@
# fix_SSLError -----
.env38/
.env39/
.vscode/
pipreqs_flake8_results.txt
pipreqs_setup_test_results.txt
pipreqs_tox_results.txt
# ------------------
*.py[cod]
# C extensions

View File

@ -66,7 +66,7 @@ Ready to contribute? Here's how to set up `pipreqs` for local development.
$ mkvirtualenv pipreqs
$ cd pipreqs/
$ python setup.py develop
$ python setup.py develop (or $pip install -e .)
4. Create a branch for local development::
@ -80,7 +80,13 @@ Ready to contribute? Here's how to set up `pipreqs` for local development.
$ python setup.py test
$ tox
To get flake8 and tox, just pip install them into your virtualenv.
To get flake8 and tox, just pip install them into your virtualenv. (or $pip install -r requirements-dev.txt)
You may also need to provide `CA_BUNDLE` as an environment variable or parameter in the `tests/.env.test` file.
$ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS
$ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS
6. Commit your changes and push your branch to GitHub::
@ -99,9 +105,9 @@ Before you submit a pull request, check that it meets these guidelines:
2. If the pull request adds functionality, the docs should be updated. Put
your new functionality into a function with a docstring, and add the
feature to the list in README.rst.
3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check
https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the
tests pass for all supported Python versions.
3. The pull request should work for Python 3.7 to 3.10 (3.11 needs work),
and PyPy3. Check https://travis-ci.org/bndr/pipreqs/pull_requests and
make sure that the tests pass for all supported Python versions.
Tips
----

View File

@ -44,6 +44,12 @@ Usage
environments parameter in your terminal:
$ export HTTP_PROXY="http://10.10.1.10:3128"
$ export HTTPS_PROXY="https://10.10.1.10:1080"
--verify <ca_bundle> Use verify to provide a CA_BUNDLE file or directory
with certificates of trusted CAs
You can also just set the environment variable in
your terminal: (`export` for nix and `set` for win)
$ export CA_BUNDLE="/certs/path/certificates.pem"
$ set CA_BUNDLE="C:/certs/path/certificates.pem"
--debug Print debug information
--ignore <dirs>... Ignore extra directories, each separated by a comma
--no-follow-links Do not follow symbolic links in the project

View File

@ -18,6 +18,12 @@ Options:
parameter in your terminal:
$ export HTTP_PROXY="http://10.10.1.10:3128"
$ export HTTPS_PROXY="https://10.10.1.10:1080"
--verify <ca_bundle> Use verify to provide a CA_BUNDLE file or directory
with certificates of trusted CAs
You can also just set the environment variable in
your terminal: (`export` for nix and `set` for win)
$ export CA_BUNDLE="/certs/path/certificates.pem" #or
$ set CA_BUNDLE="C:/certs/path/certificates.pem"
--debug Print debug information
--ignore <dirs>... Ignore extra directories, each separated by a comma
--no-follow-links Do not follow symbolic links in the project
@ -36,15 +42,17 @@ Options:
<gt> | e.g. Flask>=1.1.2
<no-pin> | e.g. Flask
"""
from contextlib import contextmanager
import os
import sys
import re
import logging
import ast
import logging
import os
import re
import sys
import traceback
from docopt import docopt
from contextlib import contextmanager
from pathlib import Path
import requests
from docopt import docopt
from yarg import json2package
from yarg.exceptions import HTTPError
@ -55,6 +63,10 @@ REGEXP = [
re.compile(r'^from ((?!\.+).*?) import (?:.*)$')
]
CA_BUNDLE = os.environ.get("CA_BUNDLE")
if CA_BUNDLE is not None:
CA_BUNDLE = str(Path(CA_BUNDLE))
@contextmanager
def _open(filename=None, mode='r'):
@ -171,7 +183,12 @@ def output_requirements(imports, symbol):
def get_imports_info(
imports, pypi_server="https://pypi.python.org/pypi/", proxy=None):
imports,
pypi_server="https://pypi.python.org/pypi/",
proxy=None,
verify=CA_BUNDLE,
c=None,
):
result = []
for item in imports:
@ -182,7 +199,10 @@ def get_imports_info(
item
)
response = requests.get(
"{0}{1}/json".format(pypi_server, item), proxies=proxy)
"{0}{1}/json".format(pypi_server, item),
proxies=proxy,
verify=verify,
)
if response.status_code == 200:
if hasattr(response.content, 'decode'):
data = json2package(response.content.decode())
@ -459,6 +479,7 @@ def init(args):
candidates = get_pkg_names(candidates)
logging.debug("Found imports: " + ", ".join(candidates))
pypi_server = "https://pypi.python.org/pypi/"
verify = CA_BUNDLE
proxy = None
if args["--pypi-server"]:
pypi_server = args["--pypi-server"]
@ -466,6 +487,9 @@ def init(args):
if args["--proxy"]:
proxy = {'http': args["--proxy"], 'https': args["--proxy"]}
if args["--verify"]:
verify = args["--verify"]
if args["--use-local"]:
logging.debug(
"Getting package information ONLY from local installation.")
@ -489,6 +513,7 @@ def init(args):
imports = local + get_imports_info(difference,
proxy=proxy,
verify=verify,
pypi_server=pypi_server)
# sort imports based on lowercase name of package, similar to `pip freeze`.
imports = sorted(imports, key=lambda x: x['name'].lower())

36
requirements-dev.txt Normal file
View File

@ -0,0 +1,36 @@
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# > python39 -m venv .env39
# # activate (.env39) virtual environment
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# upgrade pip (21.1.1 -> 22.3.1)
# (.env39)> python.exe -m pip install --upgrade pip
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# development packages
# (.env39)> pip install -r requirements-dev.txt
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# development mode installation of `pipreqs`
# (.env39)> pip install -e .
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# fix_SSLError tests.settings (optional) use of .env file
# alternative would be to set CA_BUNDLE environment variable
# $ export CA_BUNDLE="/certs/path/certificates.pem" # for nix OS
# $ set CA_BUNDLE="C:/certs/path/certificates.pem" # for win OS
python-dotenv
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# required per CONTRIBUTING workflow
flake8
tox # only needed in the environment from which tox is run
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# (optional)
# vscode settings: "python.formatting.provider": "black",
#black
docutils # reStructured Text support
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -1,3 +1,4 @@
wheel==0.38.1
Yarg==0.1.9
docopt==0.6.2
docopt==0.6.2
requests==2.28.2

View File

@ -7,7 +7,6 @@ except ImportError:
from pipreqs import __version__
with open('README.rst') as readme_file:
readme = readme_file.read()
@ -15,7 +14,10 @@ with open('HISTORY.rst') as history_file:
history = history_file.read().replace('.. :changelog:', '')
requirements = [
'docopt', 'yarg'
'docopt', 'yarg', 'requests'
]
tests_requirements = [
'python-dotenv', 'flake8'
]
setup(
@ -49,6 +51,7 @@ setup(
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
],
tests_require=tests_requirements,
test_suite='tests',
entry_points={
'console_scripts': [

16
tests/.env.test.example Normal file
View File

@ -0,0 +1,16 @@
# create a new file named `.env.test`
# and assign CA_BUNDLE to your system path\ca.pem file
CA_BUNDLE=C:\your\path\and\certificates.pem
# ----------------------------------------------------------------------------
# two alternatives to having a .env file
#
# ( 1.) you can set this value as an environment variable
# $ set CA_BUNDLE=C:\your\path\and\certificates.pem # for win OS
# $ export CA_BUNDLE=C:\your\path\and\certificates.pem # for nix OS
#
# ( 2.) or you can pass the CA_BUNDLE with the `--verify` argument
# example
# > pipreqs --verify C:\your\path\and\certificates.pem .
# ----------------------------------------------------------------------------

3
tests/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# .env.test is system specific
# do not check in to repository
.env.test

View File

@ -0,0 +1,14 @@
asposestorage
beautifulsoup4
boto
click
docopt
Flask
ipython
nose
pattern
peewee
pyflakes
requests
SQLAlchemy
ujson

29
tests/settings.py Normal file
View File

@ -0,0 +1,29 @@
"""
Environment variables can be used as a first choice
$ set CA_BUNDLE="certificates.pem" # for win OS
$ export CA_BUNDLE="certificates.pem" # for nix OS
If environment variables are not found then a second attempt
will be made by loading the values from a .env.test file in
the same directory
See ./env.test.example for details.
"""
import importlib
import os
from pathlib import Path
CA_BUNDLE = os.environ.get("CA_BUNDLE")
if CA_BUNDLE is None and importlib.find_loader("dotenv"):
# optional loading of values from .env.test file
import dotenv
env_test_path = Path(os.path.dirname(__file__) + "/.env.test")
config = dotenv.dotenv_values(env_test_path)
if config is not None:
CA_BUNDLE = config["CA_BUNDLE"]
elif CA_BUNDLE is not None:
CA_BUNDLE = str(Path(CA_BUNDLE))

View File

@ -6,14 +6,29 @@ test_pipreqs
----------------------------------
Tests for `pipreqs` module.
Environment variables used to mock arguments
e.g.,
$ set CA_BUNDLE="certificates.pem" # for win OS
$ export CA_BUNDLE="certificates.pem" # for nix OS
"""
import unittest
import os
import unittest
import requests
from pathlib import Path
from pipreqs import pipreqs
CA_BUNDLE = os.environ.get("CA_BUNDLE")
if CA_BUNDLE is None:
from tests.settings import CA_BUNDLE
else:
CA_BUNDLE = str(Path(CA_BUNDLE))
class TestPipreqs(unittest.TestCase):
@ -49,6 +64,7 @@ class TestPipreqs(unittest.TestCase):
"requirements2.txt"
)
def test_get_all_imports(self):
imports = pipreqs.get_all_imports(self.project)
self.assertEqual(len(imports), 15)
@ -80,7 +96,9 @@ class TestPipreqs(unittest.TestCase):
Test to see that the right number of packages were found on PyPI
"""
imports = pipreqs.get_all_imports(self.project)
with_info = pipreqs.get_imports_info(imports)
with_info = pipreqs.get_imports_info(
imports, proxy=None, verify=CA_BUNDLE
)
# Should contain 10 items without the "nonexistendmodule" and
# "after_method_is_valid_even_if_not_pep8"
self.assertEqual(len(with_info), 13)
@ -113,9 +131,21 @@ class TestPipreqs(unittest.TestCase):
"""
Test that all modules we will test upon are in requirements file
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
@ -130,9 +160,21 @@ class TestPipreqs(unittest.TestCase):
Test that items listed in requirements.text are the same
as locals expected
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": True,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.readlines()
@ -145,9 +187,20 @@ class TestPipreqs(unittest.TestCase):
Test that we can save requirements.txt correctly
to a different path
"""
pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path,
'--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": self.alt_requirement_path,
"--use-local": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--print": False,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.alt_requirement_path) == 1
with open(self.alt_requirement_path, "r") as f:
data = f.read().lower()
@ -163,9 +216,21 @@ class TestPipreqs(unittest.TestCase):
"""
with open(self.requirements_path, "w") as f:
f.write("should_not_be_overwritten")
pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None,
'--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
pipreqs.init(
{
"<path>": self.project,
"--savepath": None,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--print": False,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
@ -192,25 +257,39 @@ class TestPipreqs(unittest.TestCase):
Test that trying to get a custom pypi sever fails correctly
"""
self.assertRaises(
requests.exceptions.MissingSchema, pipreqs.init,
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': 'nonexistent'}
)
requests.exceptions.MissingSchema,
pipreqs.init,
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": "nonexistent",
},
)
def test_ignored_directory(self):
"""
Test --ignore parameter
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy':None, '--pypi-server':None,
'--ignore':'.ignored_dir,.ignore_second',
'--diff': None,
'--clean': None,
'--mode': None
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--ignore": ".ignored_dir,.ignore_second",
"--diff": None,
"--clean": None,
"--mode": None,
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.read().lower()
@ -222,13 +301,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=no-pin
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy': None, '--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'no-pin'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "no-pin",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.read().lower()
@ -240,14 +325,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=gt
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy': None,
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'gt'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "gt",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
@ -261,14 +351,19 @@ class TestPipreqs(unittest.TestCase):
Test --mode=compat
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy': None,
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'compat'
}
{
"<path>": self.project_with_ignore_directory,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": "compat",
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
@ -282,18 +377,36 @@ class TestPipreqs(unittest.TestCase):
Test --clean parameter
"""
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': None, '--proxy': None,
'--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": self.requirements_path,
"--mode": "non-pin",
}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
for item in self.modules[:-3]:
@ -305,19 +418,37 @@ class TestPipreqs(unittest.TestCase):
"""
cleaned_module = 'sqlalchemy'
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
{
"<path>": self.project,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": True,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": None,
"--mode": None,
}
)
assert os.path.exists(self.requirements_path) == 1
modules_clean = [m for m in self.modules if m != cleaned_module]
pipreqs.init(
{'<path>': self.project_clean, '--savepath': None,
'--print': False, '--use-local': None, '--force': None,
'--proxy': None, '--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
{
"<path>": self.project_clean,
"--savepath": None,
"--print": False,
"--use-local": None,
"--force": None,
"--proxy": None,
"--verify": CA_BUNDLE,
"--pypi-server": None,
"--diff": None,
"--clean": self.requirements_path,
"--mode": "non-pin",
}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
self.assertTrue(cleaned_module not in data)

40
tox.ini
View File

@ -1,11 +1,12 @@
[tox]
envlist = py37, py38, py39, pypy3, flake8
envlist = py37, py38, py39, py310, pypy3, flake8
[gh-actions]
python =
3.7: py37
3.8: py38
3.9: py39
3.10: py310
pypy-3.7: pypy3
[testenv]
@ -13,4 +14,39 @@ setenv =
PYTHONPATH = {toxinidir}:{toxinidir}/pipreqs
commands = python setup.py test
deps =
-r{toxinidir}/requirements.txt
-r{toxinidir}/requirements.txt
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# tox -e dev311
# Python 3.11 runtime error
# - py310 and py311 were not tested prior to this submission
# - not related to this submission (but this needs further debug to understand)
#
# ======================================================================
# FAIL: test_get_use_local_only (tests.test_pipreqs.TestPipreqs.test_get_use_local_only)
# Test without checking PyPI, check to see if names of local
# ----------------------------------------------------------------------
# Traceback (most recent call last):
# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 127, in test_get_use_local_only
# self.assertTrue(item["name"].lower() in self.local)
# AssertionError: False is not true
#
# ======================================================================
# FAIL: test_init_local_only (tests.test_pipreqs.TestPipreqs.test_init_local_only)
# Test that items listed in requirements.text are the same
# ----------------------------------------------------------------------
# Traceback (most recent call last):
# File "c:\code\ghcwm\py_dev\src\autoreq\pub\c-w-m\pipreqs\tests\test_pipreqs.py", line 182, in test_init_local_only
# self.assertTrue(item[0].lower() in self.local)
# AssertionError: False is not true
#
# ----------------------------------------------------------------------
[testenv:dev311]
description = development mode environment for debug of py311
basepython = python3.11
deps = {[testenv]deps}
# development mode for debuging
usedevelop = True
commands = python setup.py test
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~