From 8d7b7f6d1c6a741c27ec2215d419c6235556ab0c Mon Sep 17 00:00:00 2001 From: Vadim Kravcenko Date: Mon, 28 Sep 2015 20:44:43 +0200 Subject: [PATCH] feat(cli): add --proxy and --pypi-server options --- HISTORY.rst | 6 ++++++ pipreqs/__init__.py | 2 +- pipreqs/pipreqs.py | 36 +++++++++++++++++++++++++++--------- tests/test_pipreqs.py | 16 ++++++++++++---- 4 files changed, 46 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 81fea8c..23e158c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,12 @@ History ------- +0.3.0 (2015-09-29) +--------------------- + +* Add --proxy option +* Add --pypi-server option + 0.2.9 (2015-09-24) --------------------- diff --git a/pipreqs/__init__.py b/pipreqs/__init__.py index ee1ee3d..beb2a64 100755 --- a/pipreqs/__init__.py +++ b/pipreqs/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Vadim Kravcenko' __email__ = 'vadim.kravcenko@gmail.com' -__version__ = '0.2.9' +__version__ = '0.3.0' diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index f51f802..e2855dc 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -7,6 +7,8 @@ Usage: Options: --use-local Use ONLY local package info instead of querying PyPI + --pypi-server Use custom PyPi server + --proxy Use Proxy, parameter will be passed to requests library --debug Print debug information --savepath Save the list of requirements in the given file --force Overwrite existing requirements.txt @@ -18,7 +20,8 @@ import re import logging from docopt import docopt -import yarg +import requests +from yarg import json2package from yarg.exceptions import HTTPError from pipreqs import __version__ @@ -37,10 +40,6 @@ def get_all_imports(path): for root, dirs, files in os.walk(path): dirs[:] = [d for d in dirs if d not in ignore_dirs] - # for d in ignore_dirs: - # if d in dirs: - # dirs.remove(d) - candidates.append(os.path.basename(root)) files = [fn for fn in files if os.path.splitext(fn)[1] == ".py"] @@ -82,11 +81,20 @@ def generate_requirements_file(path, imports): for item in imports) + '\n') -def get_imports_info(imports): +def get_imports_info(imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): result = [] + for item in imports: try: - data = yarg.get(item) + response = requests.get("{0}{1}/json".format(pypi_server, item), proxies=proxy) + if response.status_code == 200: + if hasattr(response.content, 'decode'): + data = json2package(response.content.decode()) + else: + data = json2package(response.content) + elif response.status_code >= 300: + raise HTTPError(status_code=response.status_code, + reason=response.reason) except HTTPError: logging.debug( 'Package %s does not exist or network problems', item) @@ -157,8 +165,15 @@ def init(args): candidates = get_all_imports(args['']) candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) + pypi_server = "https://pypi.python.org/pypi/" + proxy = None + if args["--pypi-server"]: + pypi_server = args["--pypi-server"] - if args['--use-local']: + if args["--proxy"]: + proxy = {'http':args["--proxy"], 'https':args["--proxy"]} + + if args["--use-local"]: logging.debug( "Getting package information ONLY from local installation.") imports = get_import_local(candidates) @@ -168,11 +183,14 @@ def init(args): # Get packages that were not found locally difference = [x for x in candidates if x.lower() not in [z['name'].lower() for z in local]] - imports = local + get_imports_info(difference) + imports = local + get_imports_info(difference, + proxy=proxy, + pypi_server=pypi_server) path = (args["--savepath"] if args["--savepath"] else os.path.join(args[''], "requirements.txt")) + if not args["--savepath"] and not args["--force"] and os.path.exists(path): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index 38b3fe8..c7c65eb 100755 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -10,6 +10,7 @@ Tests for `pipreqs` module. import unittest import os +import requests from pipreqs import pipreqs @@ -58,7 +59,7 @@ class TestPipreqs(unittest.TestCase): def test_init(self): pipreqs.init({'': self.project, '--savepath': None, - '--use-local': None, '--force': True}) + '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -67,7 +68,7 @@ class TestPipreqs(unittest.TestCase): def test_init_local_only(self): pipreqs.init({'': self.project, '--savepath': None, - '--use-local': True, '--force': True}) + '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -77,7 +78,7 @@ class TestPipreqs(unittest.TestCase): def test_init_savepath(self): pipreqs.init({'': self.project, '--savepath': - self.alt_requirement_path, '--use-local': None}) + self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -90,7 +91,7 @@ class TestPipreqs(unittest.TestCase): with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") pipreqs.init({'': self.project, '--savepath': None, - '--use-local': None, '--force': None}) + '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -104,6 +105,13 @@ class TestPipreqs(unittest.TestCase): self.assertEqual( import_name_without_aliases, expected_import_name_without_alias) + + def test_custom_pypi_server(self): + with self.assertRaises(requests.exceptions.MissingSchema): + pipreqs.init({'': self.project, '--savepath': None, + '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':'nonexistent'}) + + def tearDown(self): try: os.remove(self.requirements_path)