diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index df6cbff..7b1f7e5 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -38,28 +38,29 @@ Options: | e.g. Flask>=1.1.2 | e.g. Flask """ -from contextlib import contextmanager -import os -import sys -import re -import logging import ast +import logging +import os +import re +import sys import traceback -from docopt import docopt +from contextlib import contextmanager + import requests +from docopt import docopt from yarg import json2package from yarg.exceptions import HTTPError from pipreqs import __version__ REGEXP = [ - re.compile(r'^import (.+)$'), - re.compile(r'^from ((?!\.+).*?) import (?:.*)$') + re.compile(r"^import (.+)$"), + re.compile(r"^from ((?!\.+).*?) import (?:.*)$"), ] @contextmanager -def _open(filename=None, mode='r'): +def _open(filename=None, mode="r"): """Open a file or ``sys.stdout`` depending on the provided filename. Args: @@ -72,13 +73,13 @@ def _open(filename=None, mode='r'): A file handle. """ - if not filename or filename == '-': - if not mode or 'r' in mode: + if not filename or filename == "-": + if not mode or "r" in mode: file = sys.stdin - elif 'w' in mode: + elif "w" in mode: file = sys.stdout else: - raise ValueError('Invalid mode for file: {}'.format(mode)) + raise ValueError("Invalid mode for file: {}".format(mode)) else: file = open(filename, mode) @@ -90,7 +91,8 @@ def _open(filename=None, mode='r'): def get_all_imports( - path, encoding=None, extra_ignore_dirs=None, follow_links=True): + path, encoding=None, extra_ignore_dirs=None, follow_links=True +): imports = set() raw_imports = set() candidates = [] @@ -139,11 +141,11 @@ def get_all_imports( # Cleanup: We only want to first part of the import. # Ex: from django.conf --> django.conf. But we only want django # as an import. - cleaned_name, _, _ = name.partition('.') + cleaned_name, _, _ = name.partition(".") imports.add(cleaned_name) packages = imports - (set(candidates) & imports) - logging.debug('Found packages: {0}'.format(packages)) + logging.debug("Found packages: {0}".format(packages)) with open(join("stdlib"), "r") as f: data = {x.strip() for x in f} @@ -157,42 +159,59 @@ def filter_line(line): def generate_requirements_file(path, imports, symbol): with _open(path, "w") as out_file: - logging.debug('Writing {num} requirements: {imports} to {file}'.format( - num=len(imports), - file=path, - imports=", ".join([x['name'] for x in imports]) - )) - fmt = '{name}' + symbol + '{version}' - out_file.write('\n'.join( - fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports) + '\n') + logging.debug( + "Writing {num} requirements: {imports} to {file}".format( + num=len(imports), + file=path, + imports=", ".join([x["name"] for x in imports]), + ) + ) + fmt = "{name}" + symbol + "{version}" + out_file.write( + "\n".join( + fmt.format(**item) + if item["version"] + else "{name}".format(**item) + for item in imports + ) + + "\n" + ) def output_requirements(imports, symbol): - generate_requirements_file('-', imports, symbol) + generate_requirements_file("-", imports, symbol) def get_imports_info( - imports, pypi_server="https://pypi.python.org/pypi/", proxy=None, verify=None): + imports, + pypi_server="https://pypi.python.org/pypi/", + proxy=None, + verify=None, +): result = [] for item in imports: try: response = requests.get( - "{0}{1}/json".format(pypi_server, item), proxies=proxy, verify=verify) + "{0}{1}/json".format(pypi_server, item), + proxies=proxy, + verify=verify, + ) if response.status_code == 200: - if hasattr(response.content, 'decode'): + if hasattr(response.content, "decode"): data = json2package(response.content.decode()) else: data = json2package(response.content) elif response.status_code >= 300: - raise HTTPError(status_code=response.status_code, - reason=response.reason) + raise HTTPError( + status_code=response.status_code, reason=response.reason + ) except HTTPError: logging.debug( - 'Package %s does not exist or network problems', item) + "Package %s does not exist or network problems", item + ) continue - result.append({'name': item, 'version': data.latest_release_id}) + result.append({"name": item, "version": data.latest_release_id}) return result @@ -212,16 +231,20 @@ def get_locally_installed_packages(encoding=None): # TODO: What errors do we intend to suppress here? continue for i_item in package_import: - if ((i_item not in ignore) and - (package[0] not in ignore)): + if (i_item not in ignore) and ( + package[0] not in ignore + ): version = None if len(package) > 1: - version = package[1].replace( - ".dist", "").replace(".egg", "") + version = ( + package[1] + .replace(".dist", "") + .replace(".egg", "") + ) packages[i_item] = { - 'version': version, - 'name': package[0] + "version": version, + "name": package[0], } return packages @@ -234,12 +257,7 @@ def get_import_local(imports, encoding=None): result.append(local[item.lower()]) # removing duplicates of package/version - result_unique = [ - dict(t) - for t in set([ - tuple(d.items()) for d in result - ]) - ] + result_unique = [dict(t) for t in set([tuple(d.items()) for d in result])] return result_unique @@ -270,7 +288,7 @@ def get_name_without_alias(name): match = REGEXP[0].match(name.strip()) if match: name = match.groups(0)[0] - return name.partition(' as ')[0].partition('.')[0].strip() + return name.partition(" as ")[0].partition(".")[0].strip() def join(f): @@ -356,7 +374,8 @@ def diff(file_, imports): logging.info( "The following modules are in {} but do not seem to be imported: " - "{}".format(file_, ", ".join(x for x in modules_not_imported))) + "{}".format(file_, ", ".join(x for x in modules_not_imported)) + ) def clean(file_, imports): @@ -404,20 +423,22 @@ def dynamic_versioning(scheme, imports): def init(args): - encoding = args.get('--encoding') - extra_ignore_dirs = args.get('--ignore') - follow_links = not args.get('--no-follow-links') - input_path = args[''] + encoding = args.get("--encoding") + extra_ignore_dirs = args.get("--ignore") + follow_links = not args.get("--no-follow-links") + input_path = args[""] if input_path is None: input_path = os.path.abspath(os.curdir) if extra_ignore_dirs: - extra_ignore_dirs = extra_ignore_dirs.split(',') + extra_ignore_dirs = extra_ignore_dirs.split(",") - candidates = get_all_imports(input_path, - encoding=encoding, - extra_ignore_dirs=extra_ignore_dirs, - follow_links=follow_links) + candidates = get_all_imports( + input_path, + encoding=encoding, + extra_ignore_dirs=extra_ignore_dirs, + follow_links=follow_links, + ) candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" @@ -427,30 +448,36 @@ def init(args): pypi_server = args["--pypi-server"] if args["--proxy"]: - proxy = {'http': args["--proxy"], 'https': args["--proxy"]} + proxy = {"http": args["--proxy"], "https": args["--proxy"]} if args["--verify"]: verify = args["--verify"] if args["--use-local"]: logging.debug( - "Getting package information ONLY from local installation.") + "Getting package information ONLY from local installation." + ) imports = get_import_local(candidates, encoding=encoding) else: logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) # Get packages that were not found locally - difference = [x for x in candidates - if x.lower() not in [z['name'].lower() for z in local]] - imports = local + get_imports_info(difference, - proxy=proxy, - verify=verify, - pypi_server=pypi_server) + difference = [ + x + for x in candidates + if x.lower() not in [z["name"].lower() for z in local] + ] + imports = local + get_imports_info( + difference, proxy=proxy, verify=verify, pypi_server=pypi_server + ) # sort imports based on lowercase name of package, similar to `pip freeze`. - imports = sorted(imports, key=lambda x: x['name'].lower()) + imports = sorted(imports, key=lambda x: x["name"].lower()) - path = (args["--savepath"] if args["--savepath"] else - os.path.join(input_path, "requirements.txt")) + path = ( + args["--savepath"] + if args["--savepath"] + else os.path.join(input_path, "requirements.txt") + ) if args["--diff"]: diff(args["--diff"], imports) @@ -460,12 +487,15 @@ def init(args): clean(args["--clean"], imports) return - if (not args["--print"] - and not args["--savepath"] - and not args["--force"] - and os.path.exists(path)): - logging.warning("requirements.txt already exists, " - "use --force to overwrite it") + if ( + not args["--print"] + and not args["--savepath"] + and not args["--force"] + and os.path.exists(path) + ): + logging.warning( + "requirements.txt already exists, " "use --force to overwrite it" + ) return if args["--mode"]: @@ -473,8 +503,10 @@ def init(args): if scheme in ["compat", "gt", "no-pin"]: imports, symbol = dynamic_versioning(scheme, imports) else: - raise ValueError("Invalid argument for mode flag, " - "use 'compat', 'gt' or 'no-pin' instead") + raise ValueError( + "Invalid argument for mode flag, " + "use 'compat', 'gt' or 'no-pin' instead" + ) else: symbol = "==" @@ -488,8 +520,8 @@ def init(args): def main(): # pragma: no cover args = docopt(__doc__, version=__version__) - log_level = logging.DEBUG if args['--debug'] else logging.INFO - logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s') + log_level = logging.DEBUG if args["--debug"] else logging.INFO + logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s") try: init(args) @@ -497,5 +529,5 @@ def main(): # pragma: no cover sys.exit(0) -if __name__ == '__main__': +if __name__ == "__main__": main() # pragma: no cover diff --git a/tests/_data/test.py b/tests/_data/test.py index fdb6ec3..73c15d2 100644 --- a/tests/_data/test.py +++ b/tests/_data/test.py @@ -1,56 +1,51 @@ """unused import""" # pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring +from __future__ import print_function + +import atexit +import curses +import importlib # html/notebookapp.py +import logging +import os +import os.path as test # [unused-import] +import signal +import sqlite3 +import sys +import time import xml.etree # [unused-import] import xml.sax # [unused-import] -import os.path as test # [unused-import] -from sys import argv as test2 # [unused-import] -from sys import flags # [unused-import] -# +1:[unused-import,unused-import] -from collections import deque, OrderedDict, Counter -# All imports above should be ignored -import requests # [unused-import] - +# astroid # setuptools -import zipimport # command/easy_install.py - +import zipimport # manager.py +# +1:[unused-import,unused-import] +from collections import Counter, OrderedDict, deque # twisted from importlib import invalidate_caches # python/test/test_deprecate.py - -# astroid -import zipimport # manager.py # IPython from importlib.machinery import all_suffixes # core/completerlib.py -import importlib # html/notebookapp.py - -from IPython.utils.importstring import import_item # Many files - -# pyflakes -# test/test_doctests.py -from pyflakes.test.test_imports import Test as TestImports - -# Nose -from nose.importer import Importer, add_path, remove_path # loader.py +from sys import argv as test2 # [unused-import] +from sys import flags # [unused-import] # see issue #88 import analytics -import flask_seasurf - -import atexit -from __future__ import print_function -from docopt import docopt -import curses, logging, sqlite3 -import logging -import os -import sqlite3 -import time -import sys -import signal +import boto as b import bs4 -import nonexistendmodule -import boto as b, peewee as p # import django import flask.ext.somext # # # +import flask_seasurf +import nonexistendmodule +import peewee as p +# All imports above should be ignored +import requests # [unused-import] +from docopt import docopt +from IPython.utils.importstring import import_item # Many files +# Nose +from nose.importer import Importer, add_path, remove_path # loader.py +# pyflakes +# test/test_doctests.py +from pyflakes.test.test_imports import Test as TestImports from sqlalchemy import model + try: import ujson as json except ImportError: @@ -62,4 +57,5 @@ import models def main(): pass + import after_method_is_valid_even_if_not_pep8 diff --git a/tests/_data_clean/test.py b/tests/_data_clean/test.py index 8cffb51..13a6d6b 100644 --- a/tests/_data_clean/test.py +++ b/tests/_data_clean/test.py @@ -1,55 +1,50 @@ """unused import""" # pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring +from __future__ import print_function + +import atexit +import curses +import importlib # html/notebookapp.py +import logging +import os +import os.path as test # [unused-import] +import signal +import sqlite3 +import sys +import time import xml.etree # [unused-import] import xml.sax # [unused-import] -import os.path as test # [unused-import] -from sys import argv as test2 # [unused-import] -from sys import flags # [unused-import] -# +1:[unused-import,unused-import] -from collections import deque, OrderedDict, Counter -# All imports above should be ignored -import requests # [unused-import] - +# astroid # setuptools -import zipimport # command/easy_install.py - +import zipimport # manager.py +# +1:[unused-import,unused-import] +from collections import Counter, OrderedDict, deque # twisted from importlib import invalidate_caches # python/test/test_deprecate.py - -# astroid -import zipimport # manager.py # IPython from importlib.machinery import all_suffixes # core/completerlib.py -import importlib # html/notebookapp.py +from sys import argv as test2 # [unused-import] +from sys import flags # [unused-import] +# see issue #88 +import analytics +import boto as b +import bs4 +# import django +import flask.ext.somext # # # +import flask_seasurf +import nonexistendmodule +import peewee as p +# All imports above should be ignored +import requests # [unused-import] +from docopt import docopt from IPython.utils.importstring import import_item # Many files - +# Nose +from nose.importer import Importer, add_path, remove_path # loader.py # pyflakes # test/test_doctests.py from pyflakes.test.test_imports import Test as TestImports -# Nose -from nose.importer import Importer, add_path, remove_path # loader.py - -# see issue #88 -import analytics -import flask_seasurf - -import atexit -from __future__ import print_function -from docopt import docopt -import curses, logging, sqlite3 -import logging -import os -import sqlite3 -import time -import sys -import signal -import bs4 -import nonexistendmodule -import boto as b, peewee as p -# import django -import flask.ext.somext # # # # from sqlalchemy import model try: import ujson as json @@ -62,4 +57,5 @@ import models def main(): pass + import after_method_is_valid_even_if_not_pep8 diff --git a/tests/_data_ignore/.ignore_second/ignored.py b/tests/_data_ignore/.ignore_second/ignored.py index b970ae3..6ea3598 100644 --- a/tests/_data_ignore/.ignore_second/ignored.py +++ b/tests/_data_ignore/.ignore_second/ignored.py @@ -1,2 +1,2 @@ # Everything in here should be ignored -from pattern.web import Twitter, plaintext \ No newline at end of file +from pattern.web import Twitter, plaintext diff --git a/tests/_data_ignore/test.py b/tests/_data_ignore/test.py index cfd039c..f520669 100644 --- a/tests/_data_ignore/test.py +++ b/tests/_data_ignore/test.py @@ -1,52 +1,48 @@ """unused import""" # pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring +from __future__ import print_function + +import atexit +import curses +import importlib # html/notebookapp.py +import logging +import os +import os.path as test # [unused-import] +import signal +import sqlite3 +import sys +import time import xml.etree # [unused-import] import xml.sax # [unused-import] -import os.path as test # [unused-import] -from sys import argv as test2 # [unused-import] -from sys import flags # [unused-import] -# +1:[unused-import,unused-import] -from collections import deque, OrderedDict, Counter -# All imports above should be ignored -import requests # [unused-import] - +# astroid # setuptools -import zipimport # command/easy_install.py - +import zipimport # manager.py +# +1:[unused-import,unused-import] +from collections import Counter, OrderedDict, deque # twisted from importlib import invalidate_caches # python/test/test_deprecate.py - -# astroid -import zipimport # manager.py # IPython from importlib.machinery import all_suffixes # core/completerlib.py -import importlib # html/notebookapp.py +from sys import argv as test2 # [unused-import] +from sys import flags # [unused-import] +import boto as b +import bs4 +# import django +import flask.ext.somext # # # +import nonexistendmodule +import peewee as p +# All imports above should be ignored +import requests # [unused-import] +from docopt import docopt from IPython.utils.importstring import import_item # Many files - +# Nose +from nose.importer import Importer, add_path, remove_path # loader.py # pyflakes # test/test_doctests.py from pyflakes.test.test_imports import Test as TestImports - -# Nose -from nose.importer import Importer, add_path, remove_path # loader.py - -import atexit -from __future__ import print_function -from docopt import docopt -import curses, logging, sqlite3 -import logging -import os -import sqlite3 -import time -import sys -import signal -import bs4 -import nonexistendmodule -import boto as b, peewee as p -# import django -import flask.ext.somext # # # from sqlalchemy import model + try: import ujson as json except ImportError: @@ -58,4 +54,5 @@ import models def main(): pass + import after_method_is_valid_even_if_not_pep8 diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index f82d3db..6e1510f 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -8,8 +8,9 @@ test_pipreqs Tests for `pipreqs` module. """ -import unittest import os +import unittest + import requests from pipreqs import pipreqs @@ -145,7 +146,7 @@ class TestPipreqs(unittest.TestCase): Test that we can save requirements.txt correctly to a different path """ - pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, + pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.alt_requirement_path) == 1 @@ -163,7 +164,7 @@ class TestPipreqs(unittest.TestCase): """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, + pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 @@ -203,7 +204,7 @@ class TestPipreqs(unittest.TestCase): Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, '--ignore':'.ignored_dir,.ignore_second', @@ -222,7 +223,7 @@ class TestPipreqs(unittest.TestCase): Test --mode=no-pin """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': None, '--diff': None,