diff --git a/.travis.yml b/.travis.yml index 324b044..7988659 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,13 @@ matrix: env: TOX_ENV=py34 - python: 2.7 env: TOX_ENV=py27 - - python: pypy - env: TOX_ENV=pypy + - python: pypy3 + env: TOX_ENV=pypy3 - python: 3.6 env: TOX_ENV=flake8 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment -install: +install: - pip install coverage coveralls tox # Command to run tests, e.g. python setup.py test diff --git a/README.rst b/README.rst index 93b1735..db37c00 100644 --- a/README.rst +++ b/README.rst @@ -70,5 +70,5 @@ Why not pip freeze? ------------------- - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. -- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv) -- and sometimes you just need to create requirements.txt for a new project without installing modules. +- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``). +- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules. diff --git a/pipreqs/mapping b/pipreqs/mapping index 6f5a469..73173b4 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -1,3 +1,4 @@ +AFQ:pyAFQ AG_fft_tools:agpy ANSI:pexpect Adafruit:Adafruit_Libraries @@ -13,6 +14,7 @@ Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper +functional:pyfunctional GeoBaseMain:GeoBasesDev GeoBases:GeoBasesDev Globals:Zope2 @@ -22,6 +24,7 @@ Kittens:astro_kittens Levenshtein:python_Levenshtein Lifetime:Zope2 MethodObject:ExtensionClass +MySQLdb:MySQL-python OFS:Zope2 OpenGL:PyOpenGL OpenSSL:pyOpenSSL @@ -592,6 +595,7 @@ devtools:tg.devtools dgis:2gis dhtmlparser:pyDHTMLParser digitalocean:python_digitalocean +discord:discord.py distribute_setup:ez_setup distutils2:Distutils2 django:Django @@ -675,6 +679,7 @@ geventwebsocket:gevent_websocket gflags:python_gflags git:GitPython github:PyGithub +github3:github3.py gitpy:git_py globusonline:globusonline_transfer_api_client google:protobuf @@ -701,7 +706,6 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad -krbV:krbv i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -727,6 +731,7 @@ keyczar:python_keyczar keyedcache:django_keyedcache keystoneclient:python_keystoneclient kickstarter:kickstart +krbv:krbV kss:kss.core kuyruk:Kuyruk langconv:AdvancedLangConv @@ -798,7 +803,6 @@ msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged -MySQL-python:MySQLdb native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -999,6 +1003,7 @@ ruamel:ruamel.base s2repoze:pysaml2 saga:saga_python saml2:pysaml2 +samtranslator:aws-sam-translator sass:libsass sassc:libsass sasstests:libsass diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 069c28a..2426933 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -369,6 +369,11 @@ def diff(file_, imports): def clean(file_, imports): """Remove modules that aren't imported in project from file.""" modules_not_imported = compare_modules(file_, imports) + + if len(modules_not_imported) == 0: + logging.info("Nothing to clean in " + file_) + return + re_remove = re.compile("|".join(modules_not_imported)) to_write = [] @@ -453,6 +458,8 @@ def init(args): imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) + # sort imports based on lowercase name of package, similar to `pip freeze`. + imports = sorted(imports, key=lambda x: x['name'].lower()) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) diff --git a/tests/_data/test.py b/tests/_data/test.py index cfd039c..fdb6ec3 100644 --- a/tests/_data/test.py +++ b/tests/_data/test.py @@ -31,6 +31,10 @@ from pyflakes.test.test_imports import Test as TestImports # Nose from nose.importer import Importer, add_path, remove_path # loader.py +# see issue #88 +import analytics +import flask_seasurf + import atexit from __future__ import print_function from docopt import docopt diff --git a/tests/_data_clean/test.py b/tests/_data_clean/test.py new file mode 100644 index 0000000..8cffb51 --- /dev/null +++ b/tests/_data_clean/test.py @@ -0,0 +1,65 @@ +"""unused import""" +# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring +import xml.etree # [unused-import] +import xml.sax # [unused-import] +import os.path as test # [unused-import] +from sys import argv as test2 # [unused-import] +from sys import flags # [unused-import] +# +1:[unused-import,unused-import] +from collections import deque, OrderedDict, Counter +# All imports above should be ignored +import requests # [unused-import] + +# setuptools +import zipimport # command/easy_install.py + +# twisted +from importlib import invalidate_caches # python/test/test_deprecate.py + +# astroid +import zipimport # manager.py +# IPython +from importlib.machinery import all_suffixes # core/completerlib.py +import importlib # html/notebookapp.py + +from IPython.utils.importstring import import_item # Many files + +# pyflakes +# test/test_doctests.py +from pyflakes.test.test_imports import Test as TestImports + +# Nose +from nose.importer import Importer, add_path, remove_path # loader.py + +# see issue #88 +import analytics +import flask_seasurf + +import atexit +from __future__ import print_function +from docopt import docopt +import curses, logging, sqlite3 +import logging +import os +import sqlite3 +import time +import sys +import signal +import bs4 +import nonexistendmodule +import boto as b, peewee as p +# import django +import flask.ext.somext # # # +# from sqlalchemy import model +try: + import ujson as json +except ImportError: + import json + +import models + + +def main(): + pass + +import after_method_is_valid_even_if_not_pep8 diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index b150c9a..c17329b 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -18,22 +18,40 @@ from pipreqs import pipreqs class TestPipreqs(unittest.TestCase): def setUp(self): - self.modules = ['flask', 'requests', 'sqlalchemy', - 'docopt', 'boto', 'ipython', 'pyflakes', 'nose', - 'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ] + self.modules = [ + 'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython', + 'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee', + 'ujson', 'nonexistendmodule', 'bs4', + 'after_method_is_valid_even_if_not_pep8' + ] self.modules2 = ['beautifulsoup4'] self.local = ["docopt", "requests", "nose", 'pyflakes'] self.project = os.path.join(os.path.dirname(__file__), "_data") - self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data") - self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore") - self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps") + self.project_clean = os.path.join( + os.path.dirname(__file__), + "_data_clean" + ) + self.project_invalid = os.path.join( + os.path.dirname(__file__), + "_invalid_data" + ) + self.project_with_ignore_directory = os.path.join( + os.path.dirname(__file__), + "_data_ignore" + ) + self.project_with_duplicated_deps = os.path.join( + os.path.dirname(__file__), + "_data_duplicated_deps" + ) self.requirements_path = os.path.join(self.project, "requirements.txt") self.alt_requirement_path = os.path.join( - self.project, "requirements2.txt") + self.project, + "requirements2.txt" + ) def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) - self.assertEqual(len(imports), 13) + self.assertEqual(len(imports), 15) for item in imports: self.assertTrue( item.lower() in self.modules, "Import is missing: " + item) @@ -54,7 +72,8 @@ class TestPipreqs(unittest.TestCase): """ Test that invalid python files cannot be imported. """ - self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid) + self.assertRaises( + SyntaxError, pipreqs.get_all_imports, self.project_invalid) def test_get_imports_info(self): """ @@ -62,8 +81,9 @@ class TestPipreqs(unittest.TestCase): """ imports = pipreqs.get_all_imports(self.project) with_info = pipreqs.get_imports_info(imports) - # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8" - self.assertEqual(len(with_info), 11) + # Should contain 10 items without the "nonexistendmodule" and + # "after_method_is_valid_even_if_not_pep8" + self.assertEqual(len(with_info), 13) for item in with_info: self.assertTrue( item['name'].lower() in self.modules, @@ -77,10 +97,12 @@ class TestPipreqs(unittest.TestCase): def test_get_use_local_only(self): """ - Test without checking PyPI, check to see if names of local imports matches what we expect + Test without checking PyPI, check to see if names of local + imports matches what we expect - Note even though pyflakes isn't in requirements.txt, - It's added to locals since it is a development dependency for testing + It's added to locals since it is a development dependency + for testing """ # should find only docopt and requests imports_with_info = pipreqs.get_import_local(self.modules) @@ -89,24 +111,32 @@ class TestPipreqs(unittest.TestCase): def test_init(self): """ - Test that all modules we will test upon, are in requirements file + Test that all modules we will test upon are in requirements file """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--dynamic': None}) + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None} + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: self.assertTrue(item.lower() in data) + # It should be sorted based on names. + data = data.strip().split('\n') + self.assertEqual(data, sorted(data)) def test_init_local_only(self): """ - Test that items listed in requirements.text are the same as locals expected + Test that items listed in requirements.text are the same + as locals expected """ - pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--dynamic': None}) + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': True, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None} + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -116,11 +146,14 @@ class TestPipreqs(unittest.TestCase): def test_init_savepath(self): """ - Test that we can save requiremnts.tt correctly to a different path + Test that we can save requirements.txt correctly + to a different path """ - pipreqs.init({'': self.project, '--savepath': - self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None, '--dynamic': None}) + pipreqs.init( + {'': self.project, '--savepath': self.alt_requirement_path, + '--use-local': None, '--proxy': None, '--pypi-server': None, + '--print': False, "--diff": None, "--clean": None} + ) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -131,13 +164,16 @@ class TestPipreqs(unittest.TestCase): def test_init_overwrite(self): """ - Test that if requiremnts.txt exists, it will not automatically be overwritten + Test that if requiremnts.txt exists, it will not be + automatically overwritten """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, - '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None, '--dynamic': None}) + pipreqs.init( + {'': self.project, '--savepath': None, '--use-local': None, + '--force': None, '--proxy': None, '--pypi-server': None, + '--print': False, '--diff': None, '--clean': None} + ) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -145,39 +181,48 @@ class TestPipreqs(unittest.TestCase): def test_get_import_name_without_alias(self): """ - Test that function get_name_without_alias() will work on a string. - - Note: This isn't truly needed when pipreqs is walking the AST to find imports + Test that function get_name_without_alias() + will work on a string. + - Note: This isn't truly needed when pipreqs is walking + the AST to find imports """ import_name_with_alias = "requests as R" expected_import_name_without_alias = "requests" import_name_without_aliases = pipreqs.get_name_without_alias( import_name_with_alias) self.assertEqual( - import_name_without_aliases, expected_import_name_without_alias) + import_name_without_aliases, + expected_import_name_without_alias + ) def test_custom_pypi_server(self): """ Test that trying to get a custom pypi sever fails correctly """ - self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'}) + self.assertRaises( + requests.exceptions.MissingSchema, pipreqs.init, + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': 'nonexistent'} + ) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy':None, - '--pypi-server':None, - '--ignore':'.ignored_dir,.ignore_second', - '--diff': None, - '--clean': None, - '--dynamic': None - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + {'': self.project_with_ignore_directory, '--savepath': None, + '--print': False, '--use-local': None, '--force': True, + '--proxy': None, '--pypi-server': None, + '--ignore': '.ignored_dir,.ignore_second', '--diff': None, + '--clean': None} + ) + with open( + os.path.join( + self.project_with_ignore_directory, + "requirements.txt" + ), "r" + ) as f: data = f.read().lower() for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) @@ -187,16 +232,15 @@ class TestPipreqs(unittest.TestCase): Test --dynamic=all """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--dynamic': 'all' - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + {'': self.project_with_ignore_directory, '--savepath': None, + '--print': False, '--use-local': None, '--force': True, + '--proxy': None, '--pypi-server': None, '--diff': None, + '--clean': None, '--no-pin': True} + ) + with open(os.path.join( + self.project_with_ignore_directory, + "requirements.txt"), "r" + ) as f: data = f.read().lower() for item in ['beautifulsoup4', 'boto']: self.assertTrue(item.lower() in data) @@ -241,6 +285,49 @@ class TestPipreqs(unittest.TestCase): asterisk = item.strip().split(".")[1] self.assertEqual(asterisk, "*",) + def test_clean(self): + """ + Test --clean parameter + """ + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None} + ) + assert os.path.exists(self.requirements_path) == 1 + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': None, '--proxy': None, + '--pypi-server': None, '--diff': None, + '--clean': self.requirements_path, '--no-pin': True} + ) + with open(self.requirements_path, "r") as f: + data = f.read().lower() + for item in self.modules[:-3]: + self.assertTrue(item.lower() in data) + + def test_clean_with_imports_to_clean(self): + """ + Test --clean parameter when there are imports to clean + """ + cleaned_module = 'sqlalchemy' + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None} + ) + assert os.path.exists(self.requirements_path) == 1 + modules_clean = [m for m in self.modules if m != cleaned_module] + pipreqs.init( + {'': self.project_clean, '--savepath': None, + '--print': False, '--use-local': None, '--force': None, + '--proxy': None, '--pypi-server': None, '--diff': None, + '--clean': self.requirements_path, '--no-pin': True} + ) + with open(self.requirements_path, "r") as f: + data = f.read().lower() + self.assertTrue(cleaned_module not in data) + def tearDown(self): """ Remove requiremnts.txt files that were written diff --git a/tox.ini b/tox.ini index 28a1dfa..554b3c1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy, flake8 +envlist = py27, py34, py35, py36, pypy3, flake8 [testenv] setenv =