merged from origin

.travis.yml

@@ -12,13 +12,13 @@ matrix:
       env: TOX_ENV=py34
     - python: 2.7
       env: TOX_ENV=py27
-    - python: pypy
+    - python: pypy3
-      env: TOX_ENV=pypy
+      env: TOX_ENV=pypy3
     - python: 3.6
       env: TOX_ENV=flake8
 
 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment
-install: 
+install:
   - pip install coverage coveralls tox
 
 # Command to run tests, e.g. python setup.py test

README.rst

@@ -70,5 +70,5 @@ Why not pip freeze?
 -------------------
 
 - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. 
-- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv)
+- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``).
-- and sometimes you just need to create requirements.txt for a new project without installing modules.
+- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules.

pipreqs/mapping

@@ -1,3 +1,4 @@
+AFQ:pyAFQ
 AG_fft_tools:agpy
 ANSI:pexpect
 Adafruit:Adafruit_Libraries
@@ -13,6 +14,7 @@ Crypto:pycryptodome
 Cryptodome:pycryptodomex
 FSM:pexpect
 FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper
+functional:pyfunctional
 GeoBaseMain:GeoBasesDev
 GeoBases:GeoBasesDev
 Globals:Zope2
@@ -22,6 +24,7 @@ Kittens:astro_kittens
 Levenshtein:python_Levenshtein
 Lifetime:Zope2
 MethodObject:ExtensionClass
+MySQLdb:MySQL-python
 OFS:Zope2
 OpenGL:PyOpenGL
 OpenSSL:pyOpenSSL
@@ -592,6 +595,7 @@ devtools:tg.devtools
 dgis:2gis
 dhtmlparser:pyDHTMLParser
 digitalocean:python_digitalocean
+discord:discord.py
 distribute_setup:ez_setup
 distutils2:Distutils2
 django:Django
@@ -675,6 +679,7 @@ geventwebsocket:gevent_websocket
 gflags:python_gflags
 git:GitPython
 github:PyGithub
+github3:github3.py
 gitpy:git_py
 globusonline:globusonline_transfer_api_client
 google:protobuf
@@ -701,7 +706,6 @@ html:pies2overrides
 htmloutput:nosehtmloutput
 http:pies2overrides
 hvad:django_hvad
-krbV:krbv
 i99fix:199Fix
 igraph:python_igraph
 imdb:IMDbPY
@@ -727,6 +731,7 @@ keyczar:python_keyczar
 keyedcache:django_keyedcache
 keystoneclient:python_keystoneclient
 kickstarter:kickstart
+krbv:krbV
 kss:kss.core
 kuyruk:Kuyruk
 langconv:AdvancedLangConv
@@ -798,7 +803,6 @@ msgpack:msgpack_python
 mutations:aino_mutations
 mws:amazon_mws
 mysql:mysql_connector_repackaged
-MySQL-python:MySQLdb
 native_tags:django_native_tags
 ndg:ndg_httpsclient
 nereid:trytond_nereid
@@ -999,6 +1003,7 @@ ruamel:ruamel.base
 s2repoze:pysaml2
 saga:saga_python
 saml2:pysaml2
+samtranslator:aws-sam-translator
 sass:libsass
 sassc:libsass
 sasstests:libsass

pipreqs/pipreqs.py

@@ -369,6 +369,11 @@ def diff(file_, imports):
 def clean(file_, imports):
     """Remove modules that aren't imported in project from file."""
     modules_not_imported = compare_modules(file_, imports)
+
+    if len(modules_not_imported) == 0:
+        logging.info("Nothing to clean in " + file_)
+        return
+
     re_remove = re.compile("|".join(modules_not_imported))
     to_write = []
 
@@ -453,6 +458,8 @@ def init(args):
         imports = local + get_imports_info(difference,
                                            proxy=proxy,
                                            pypi_server=pypi_server)
+    # sort imports based on lowercase name of package, similar to `pip freeze`.
+    imports = sorted(imports, key=lambda x: x['name'].lower())
 
     path = (args["--savepath"] if args["--savepath"] else
             os.path.join(input_path, "requirements.txt"))

tests/_data/test.py

@@ -31,6 +31,10 @@ from pyflakes.test.test_imports import Test as TestImports
 # Nose
 from nose.importer import Importer, add_path, remove_path  # loader.py
 
+# see issue #88
+import analytics
+import flask_seasurf
+
 import atexit
 from __future__ import print_function
 from docopt import docopt

tests/_data_clean/test.py

@@ -0,0 +1,65 @@
+"""unused import"""
+# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
+import xml.etree  # [unused-import]
+import xml.sax  # [unused-import]
+import os.path as test  # [unused-import]
+from sys import argv as test2  # [unused-import]
+from sys import flags  # [unused-import]
+# +1:[unused-import,unused-import]
+from collections import deque, OrderedDict, Counter
+# All imports above should be ignored
+import requests  # [unused-import]
+
+# setuptools
+import zipimport  # command/easy_install.py
+
+# twisted
+from importlib import invalidate_caches  # python/test/test_deprecate.py
+
+# astroid
+import zipimport  # manager.py
+# IPython
+from importlib.machinery import all_suffixes  # core/completerlib.py
+import importlib  # html/notebookapp.py
+
+from IPython.utils.importstring import import_item  # Many files
+
+# pyflakes
+# test/test_doctests.py
+from pyflakes.test.test_imports import Test as TestImports
+
+# Nose
+from nose.importer import Importer, add_path, remove_path  # loader.py
+
+# see issue #88
+import analytics
+import flask_seasurf
+
+import atexit
+from __future__ import print_function
+from docopt import docopt
+import curses, logging, sqlite3
+import logging
+import os
+import sqlite3
+import time
+import sys
+import signal
+import bs4
+import nonexistendmodule
+import boto as b, peewee as p
+# import django
+import flask.ext.somext  # # #
+# from sqlalchemy import model
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+import models
+
+
+def main():
+    pass
+
+import after_method_is_valid_even_if_not_pep8

tests/test_pipreqs.py

@@ -18,22 +18,40 @@ from pipreqs import pipreqs
 class TestPipreqs(unittest.TestCase):
 
     def setUp(self):
-        self.modules = ['flask', 'requests', 'sqlalchemy',
+        self.modules = [
-                        'docopt', 'boto', 'ipython', 'pyflakes', 'nose',
+            'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython',
-                        'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ]
+            'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee',
+            'ujson', 'nonexistendmodule', 'bs4',
+            'after_method_is_valid_even_if_not_pep8'
+            ]
         self.modules2 = ['beautifulsoup4']
         self.local = ["docopt", "requests", "nose", 'pyflakes']
         self.project = os.path.join(os.path.dirname(__file__), "_data")
-        self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data")
+        self.project_clean = os.path.join(
-        self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore")
+            os.path.dirname(__file__),
-        self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps")
+            "_data_clean"
+            )
+        self.project_invalid = os.path.join(
+            os.path.dirname(__file__),
+            "_invalid_data"
+            )
+        self.project_with_ignore_directory = os.path.join(
+            os.path.dirname(__file__),
+            "_data_ignore"
+            )
+        self.project_with_duplicated_deps = os.path.join(
+            os.path.dirname(__file__),
+            "_data_duplicated_deps"
+            )
         self.requirements_path = os.path.join(self.project, "requirements.txt")
         self.alt_requirement_path = os.path.join(
-            self.project, "requirements2.txt")
+            self.project,
+            "requirements2.txt"
+            )
 
     def test_get_all_imports(self):
         imports = pipreqs.get_all_imports(self.project)
-        self.assertEqual(len(imports), 13)
+        self.assertEqual(len(imports), 15)
         for item in imports:
             self.assertTrue(
                 item.lower() in self.modules, "Import is missing: " + item)
@@ -54,7 +72,8 @@ class TestPipreqs(unittest.TestCase):
         """
         Test that invalid python files cannot be imported.
         """
-        self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
+        self.assertRaises(
+            SyntaxError, pipreqs.get_all_imports, self.project_invalid)
 
     def test_get_imports_info(self):
         """
@@ -62,8 +81,9 @@ class TestPipreqs(unittest.TestCase):
         """
         imports = pipreqs.get_all_imports(self.project)
         with_info = pipreqs.get_imports_info(imports)
-        # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8"
+        # Should contain 10 items without the "nonexistendmodule" and
-        self.assertEqual(len(with_info), 11)
+        # "after_method_is_valid_even_if_not_pep8"
+        self.assertEqual(len(with_info), 13)
         for item in with_info:
             self.assertTrue(
                 item['name'].lower() in self.modules,
@@ -77,10 +97,12 @@ class TestPipreqs(unittest.TestCase):
 
     def test_get_use_local_only(self):
         """
-        Test without checking PyPI, check to see if names of local imports matches what we expect
+        Test without checking PyPI, check to see if names of local
+        imports matches what we expect
 
         - Note even though pyflakes isn't in requirements.txt,
-          It's added to locals since it is a development dependency for testing
+          It's added to locals since it is a development dependency
+          for testing
         """
         # should find only docopt and requests
         imports_with_info = pipreqs.get_import_local(self.modules)
@@ -89,24 +111,32 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init(self):
         """
-        Test that all modules we will test upon, are in requirements file
+        Test that all modules we will test upon are in requirements file
         """
-        pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
+        pipreqs.init(
-                      '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None,
+            {'<path>': self.project, '--savepath': None, '--print': False,
-                      '--diff': None, '--clean': None, '--dynamic': None})
+             '--use-local': None, '--force': True, '--proxy': None,
+             '--pypi-server': None, '--diff': None, '--clean': None}
+            )
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.read().lower()
             for item in self.modules[:-3]:
                 self.assertTrue(item.lower() in data)
+        # It should be sorted based on names.
+        data = data.strip().split('\n')
+        self.assertEqual(data, sorted(data))
 
     def test_init_local_only(self):
         """
-        Test that items listed in requirements.text are the same as locals expected
+        Test that items listed in requirements.text are the same
+        as locals expected
         """
-        pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
+        pipreqs.init(
-                      '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None,
+            {'<path>': self.project, '--savepath': None, '--print': False,
-                      '--diff': None, '--clean': None, '--dynamic': None})
+             '--use-local': True, '--force': True, '--proxy': None,
+             '--pypi-server': None, '--diff': None, '--clean': None}
+            )
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.readlines()
@@ -116,11 +146,14 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init_savepath(self):
         """
-        Test that we can save requiremnts.tt correctly to a different path
+        Test that we can save requirements.txt correctly
+        to a different path
         """
-        pipreqs.init({'<path>': self.project, '--savepath':
+        pipreqs.init(
-                      self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None,  '--print': False,
+            {'<path>': self.project, '--savepath': self.alt_requirement_path,
-                      "--diff": None, "--clean": None, '--dynamic': None})
+             '--use-local': None, '--proxy': None, '--pypi-server': None,
+             '--print': False, "--diff": None, "--clean": None}
+            )
         assert os.path.exists(self.alt_requirement_path) == 1
         with open(self.alt_requirement_path, "r") as f:
             data = f.read().lower()
@@ -131,13 +164,16 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init_overwrite(self):
         """
-        Test that if requiremnts.txt exists, it will not automatically be overwritten
+        Test that if requiremnts.txt exists, it will not be
+        automatically overwritten
         """
         with open(self.requirements_path, "w") as f:
             f.write("should_not_be_overwritten")
-        pipreqs.init({'<path>': self.project, '--savepath': None,
+        pipreqs.init(
-                      '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
+            {'<path>': self.project, '--savepath': None, '--use-local': None,
-                      "--diff": None, "--clean": None, '--dynamic': None})
+             '--force': None, '--proxy': None, '--pypi-server': None,
+             '--print': False, '--diff': None, '--clean': None}
+            )
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.read().lower()
@@ -145,39 +181,48 @@ class TestPipreqs(unittest.TestCase):
 
     def test_get_import_name_without_alias(self):
         """
-        Test that function get_name_without_alias() will work on a string.
+        Test that function get_name_without_alias()
-        - Note: This isn't truly needed when pipreqs is walking the AST to find imports
+        will work on a string.
+        - Note: This isn't truly needed when pipreqs is walking
+          the AST to find imports
         """
         import_name_with_alias = "requests as R"
         expected_import_name_without_alias = "requests"
         import_name_without_aliases = pipreqs.get_name_without_alias(
             import_name_with_alias)
         self.assertEqual(
-            import_name_without_aliases, expected_import_name_without_alias)
+            import_name_without_aliases,
+            expected_import_name_without_alias
+            )
 
     def test_custom_pypi_server(self):
         """
         Test that trying to get a custom pypi sever fails correctly
         """
-        self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'<path>': self.project, '--savepath': None, '--print': False,
+        self.assertRaises(
-                          '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'})
+            requests.exceptions.MissingSchema, pipreqs.init,
+            {'<path>': self.project, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True, '--proxy': None,
+             '--pypi-server': 'nonexistent'}
+            )
 
     def test_ignored_directory(self):
         """
         Test --ignore parameter
         """
         pipreqs.init(
-            {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
+            {'<path>': self.project_with_ignore_directory, '--savepath': None,
-                      '--use-local': None, '--force': True,
+             '--print': False, '--use-local': None, '--force': True,
-                      '--proxy':None,
+             '--proxy': None, '--pypi-server': None,
-                      '--pypi-server':None,
+             '--ignore': '.ignored_dir,.ignore_second', '--diff': None,
-                      '--ignore':'.ignored_dir,.ignore_second',
+             '--clean': None}
-                      '--diff': None,
+            )
-                      '--clean': None,
+        with open(
-                      '--dynamic': None
+                  os.path.join(
-             }
+                    self.project_with_ignore_directory,
-        )
+                    "requirements.txt"
-        with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
+                    ), "r"
+                  ) as f:
             data = f.read().lower()
             for item in ['click', 'getpass']:
                 self.assertFalse(item.lower() in data)
@@ -187,16 +232,15 @@ class TestPipreqs(unittest.TestCase):
         Test --dynamic=all
         """
         pipreqs.init(
-            {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
+            {'<path>': self.project_with_ignore_directory, '--savepath': None,
-             '--use-local': None, '--force': True,
+             '--print': False, '--use-local': None, '--force': True,
-             '--proxy': None,
+             '--proxy': None, '--pypi-server': None, '--diff': None,
-             '--pypi-server': None,
+             '--clean': None, '--no-pin': True}
-             '--diff': None,
+            )
-             '--clean': None,
+        with open(os.path.join(
-             '--dynamic': 'all'
+            self.project_with_ignore_directory,
-             }
+            "requirements.txt"), "r"
-        )
+            ) as f:
-        with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
             data = f.read().lower()
             for item in ['beautifulsoup4', 'boto']:
                 self.assertTrue(item.lower() in data)
@@ -241,6 +285,49 @@ class TestPipreqs(unittest.TestCase):
                 asterisk = item.strip().split(".")[1]
                 self.assertEqual(asterisk, "*",)
 
+    def test_clean(self):
+        """
+        Test --clean parameter
+        """
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True, '--proxy': None,
+             '--pypi-server': None, '--diff': None, '--clean': None}
+            )
+        assert os.path.exists(self.requirements_path) == 1
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+            '--use-local': None, '--force': None, '--proxy': None,
+            '--pypi-server': None, '--diff': None,
+            '--clean': self.requirements_path, '--no-pin': True}
+            )
+        with open(self.requirements_path, "r") as f:
+            data = f.read().lower()
+            for item in self.modules[:-3]:
+                self.assertTrue(item.lower() in data)
+
+    def test_clean_with_imports_to_clean(self):
+        """
+        Test --clean parameter when there are imports to clean
+        """
+        cleaned_module = 'sqlalchemy'
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+            '--use-local': None, '--force': True, '--proxy': None,
+            '--pypi-server': None, '--diff': None, '--clean': None}
+            )
+        assert os.path.exists(self.requirements_path) == 1
+        modules_clean = [m for m in self.modules if m != cleaned_module]
+        pipreqs.init(
+            {'<path>': self.project_clean, '--savepath': None,
+            '--print': False, '--use-local': None, '--force': None,
+            '--proxy': None, '--pypi-server': None, '--diff': None,
+            '--clean': self.requirements_path, '--no-pin': True}
+            )
+        with open(self.requirements_path, "r") as f:
+            data = f.read().lower()
+            self.assertTrue(cleaned_module not in data)
+
     def tearDown(self):
         """
         Remove requiremnts.txt files that were written

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py34, py35, py36, pypy, flake8
+envlist = py27, py34, py35, py36, pypy3, flake8
 
 [testenv]
 setenv =