Revert "Revert all commits since last release"

This reverts commit 90102acdbb23c09574d27df8bd1f568d34e0cfd3.
Now that we are ready to make a new release we can revert the revert and
hopefuly never have to solve a mess like this again to keep master
synchronized with the latest release

.travis.yml

@@ -12,8 +12,8 @@ matrix:
       env: TOX_ENV=py34
     - python: 2.7
       env: TOX_ENV=py27
-    - python: pypy
+    - python: pypy3
-      env: TOX_ENV=pypy
+      env: TOX_ENV=pypy3
     - python: 3.6
       env: TOX_ENV=flake8
 

AUTHORS.rst

@@ -10,4 +10,5 @@ Development Lead
 Contributors
 ------------
 
-None yet. Why not be the first?
+* Jake Teo <mapattacker@gmail.com>
+* Jerome Chan <cjerome94@gmail.com>

HISTORY.rst

@@ -3,6 +3,11 @@
 History
 -------
 
+0.4.11 (2020-03-29)
+--------------------
+
+* Implement '--mode' (Jake Teo, Jerome Chan)
+
 0.4.8 (2017-06-30)
 --------------------
 

README.rst

@@ -49,7 +49,10 @@ Usage
         --force               Overwrite existing requirements.txt
         --diff <file>         Compare modules in requirements.txt to project imports.
         --clean <file>        Clean up requirements.txt by removing modules that are not imported in project.
-        --no-pin              Omit version of output packages.
+        --mode <scheme>       Enables dynamic versioning with <compat>, <gt> or <non-pin> schemes.
+                              <compat> | e.g. Flask~=1.1.2
+                              <gt>     | e.g. Flask>=1.1.2
+                              <no-pin> | e.g. Flask
 Example
 -------
 
@@ -70,5 +73,5 @@ Why not pip freeze?
 -------------------
 
 - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. 
-- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv)
+- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``).
-- and sometimes you just need to create requirements.txt for a new project without installing modules.
+- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules.

pipreqs/mapping

@@ -1,3 +1,4 @@
+AFQ:pyAFQ
 AG_fft_tools:agpy
 ANSI:pexpect
 Adafruit:Adafruit_Libraries
@@ -13,6 +14,7 @@ Crypto:pycryptodome
 Cryptodome:pycryptodomex
 FSM:pexpect
 FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper
+functional:pyfunctional
 GeoBaseMain:GeoBasesDev
 GeoBases:GeoBasesDev
 Globals:Zope2
@@ -22,6 +24,7 @@ Kittens:astro_kittens
 Levenshtein:python_Levenshtein
 Lifetime:Zope2
 MethodObject:ExtensionClass
+MySQLdb:MySQL-python
 OFS:Zope2
 OpenGL:PyOpenGL
 OpenSSL:pyOpenSSL
@@ -592,6 +595,7 @@ devtools:tg.devtools
 dgis:2gis
 dhtmlparser:pyDHTMLParser
 digitalocean:python_digitalocean
+discord:discord.py
 distribute_setup:ez_setup
 distutils2:Distutils2
 django:Django
@@ -675,6 +679,7 @@ geventwebsocket:gevent_websocket
 gflags:python_gflags
 git:GitPython
 github:PyGithub
+github3:github3.py
 gitpy:git_py
 globusonline:globusonline_transfer_api_client
 google:protobuf
@@ -701,7 +706,7 @@ html:pies2overrides
 htmloutput:nosehtmloutput
 http:pies2overrides
 hvad:django_hvad
-krbV:krbv
+hydra:hydra-core
 i99fix:199Fix
 igraph:python_igraph
 imdb:IMDbPY
@@ -727,6 +732,7 @@ keyczar:python_keyczar
 keyedcache:django_keyedcache
 keystoneclient:python_keystoneclient
 kickstarter:kickstart
+krbv:krbV
 kss:kss.core
 kuyruk:Kuyruk
 langconv:AdvancedLangConv
@@ -798,7 +804,6 @@ msgpack:msgpack_python
 mutations:aino_mutations
 mws:amazon_mws
 mysql:mysql_connector_repackaged
-MySQL-python:MySQLdb
 native_tags:django_native_tags
 ndg:ndg_httpsclient
 nereid:trytond_nereid
@@ -999,6 +1004,7 @@ ruamel:ruamel.base
 s2repoze:pysaml2
 saga:saga_python
 saml2:pysaml2
+samtranslator:aws-sam-translator
 sass:libsass
 sassc:libsass
 sasstests:libsass

pipreqs/pipreqs.py

@@ -31,7 +31,11 @@ Options:
                           imports.
     --clean <file>        Clean up requirements.txt by removing modules
                           that are not imported in project.
-    --no-pin              Omit version of output packages.
+    --mode <scheme>       Enables dynamic versioning with <compat>,
+                          <gt> or <non-pin> schemes.
+                          <compat> | e.g. Flask~=1.1.2
+                          <gt>     | e.g. Flask>=1.1.2
+                          <no-pin> | e.g. Flask
 """
 from __future__ import print_function, absolute_import
 from contextlib import contextmanager
@@ -157,25 +161,25 @@ def get_all_imports(
     return list(packages - data)
 
 
-def filter_line(l):
+def filter_line(line):
-    return len(l) > 0 and l[0] != "#"
+    return len(line) > 0 and line[0] != "#"
 
 
-def generate_requirements_file(path, imports):
+def generate_requirements_file(path, imports, symbol):
     with _open(path, "w") as out_file:
         logging.debug('Writing {num} requirements: {imports} to {file}'.format(
             num=len(imports),
             file=path,
             imports=", ".join([x['name'] for x in imports])
         ))
-        fmt = '{name}=={version}'
+        fmt = '{name}' + symbol + '{version}'
         out_file.write('\n'.join(
             fmt.format(**item) if item['version'] else '{name}'.format(**item)
             for item in imports) + '\n')
 
 
-def output_requirements(imports):
+def output_requirements(imports, symbol):
-    generate_requirements_file('-', imports)
+    generate_requirements_file('-', imports, symbol)
 
 
 def get_imports_info(
@@ -368,6 +372,11 @@ def diff(file_, imports):
 def clean(file_, imports):
     """Remove modules that aren't imported in project from file."""
     modules_not_imported = compare_modules(file_, imports)
+
+    if len(modules_not_imported) == 0:
+        logging.info("Nothing to clean in " + file_)
+        return
+
     re_remove = re.compile("|".join(modules_not_imported))
     to_write = []
 
@@ -392,6 +401,18 @@ def clean(file_, imports):
     logging.info("Successfully cleaned up requirements in " + file_)
 
 
+def dynamic_versioning(scheme, imports):
+    """Enables dynamic versioning with <compat>, <gt> or <non-pin> schemes."""
+    if scheme == "no-pin":
+        imports = [{"name": item["name"], "version": ""} for item in imports]
+        symbol = ""
+    elif scheme == "gt":
+        symbol = ">="
+    elif scheme == "compat":
+        symbol = "~="
+    return imports, symbol
+
+
 def init(args):
     encoding = args.get('--encoding')
     extra_ignore_dirs = args.get('--ignore')
@@ -430,6 +451,8 @@ def init(args):
         imports = local + get_imports_info(difference,
                                            proxy=proxy,
                                            pypi_server=pypi_server)
+    # sort imports based on lowercase name of package, similar to `pip freeze`.
+    imports = sorted(imports, key=lambda x: x['name'].lower())
 
     path = (args["--savepath"] if args["--savepath"] else
             os.path.join(input_path, "requirements.txt"))
@@ -446,18 +469,25 @@ def init(args):
             and not args["--savepath"]
             and not args["--force"]
             and os.path.exists(path)):
-        logging.warning("Requirements.txt already exists, "
+        logging.warning("requirements.txt already exists, "
                         "use --force to overwrite it")
         return
 
-    if args.get('--no-pin'):
+    if args["--mode"]:
-        imports = [{'name': item["name"], 'version': ''} for item in imports]
+        scheme = args.get("--mode")
+        if scheme in ["compat", "gt", "no-pin"]:
+            imports, symbol = dynamic_versioning(scheme, imports)
+        else:
+            raise ValueError("Invalid argument for mode flag, "
+                             "use 'compat', 'gt' or 'no-pin' instead")
+    else:
+        symbol = "=="
 
     if args["--print"]:
-        output_requirements(imports)
+        output_requirements(imports, symbol)
         logging.info("Successfully output requirements")
     else:
-        generate_requirements_file(path, imports)
+        generate_requirements_file(path, imports, symbol)
         logging.info("Successfully saved requirements file in " + path)
 
 

pipreqs/stdlib

@@ -133,6 +133,7 @@ curses
 curses.ascii
 curses.panel
 curses.textpad
+dataclasses
 datetime
 dbhash
 dbm
@@ -370,6 +371,7 @@ robotparser
 runpy
 sched
 ScrolledText
+secrets
 select
 selectors
 sets

tests/_data/test.py

@@ -31,6 +31,10 @@ from pyflakes.test.test_imports import Test as TestImports
 # Nose
 from nose.importer import Importer, add_path, remove_path  # loader.py
 
+# see issue #88
+import analytics
+import flask_seasurf
+
 import atexit
 from __future__ import print_function
 from docopt import docopt

tests/_data_clean/test.py

@@ -0,0 +1,65 @@
+"""unused import"""
+# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
+import xml.etree  # [unused-import]
+import xml.sax  # [unused-import]
+import os.path as test  # [unused-import]
+from sys import argv as test2  # [unused-import]
+from sys import flags  # [unused-import]
+# +1:[unused-import,unused-import]
+from collections import deque, OrderedDict, Counter
+# All imports above should be ignored
+import requests  # [unused-import]
+
+# setuptools
+import zipimport  # command/easy_install.py
+
+# twisted
+from importlib import invalidate_caches  # python/test/test_deprecate.py
+
+# astroid
+import zipimport  # manager.py
+# IPython
+from importlib.machinery import all_suffixes  # core/completerlib.py
+import importlib  # html/notebookapp.py
+
+from IPython.utils.importstring import import_item  # Many files
+
+# pyflakes
+# test/test_doctests.py
+from pyflakes.test.test_imports import Test as TestImports
+
+# Nose
+from nose.importer import Importer, add_path, remove_path  # loader.py
+
+# see issue #88
+import analytics
+import flask_seasurf
+
+import atexit
+from __future__ import print_function
+from docopt import docopt
+import curses, logging, sqlite3
+import logging
+import os
+import sqlite3
+import time
+import sys
+import signal
+import bs4
+import nonexistendmodule
+import boto as b, peewee as p
+# import django
+import flask.ext.somext  # # #
+# from sqlalchemy import model
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+import models
+
+
+def main():
+    pass
+
+import after_method_is_valid_even_if_not_pep8

tests/test_pipreqs.py

@@ -18,22 +18,40 @@ from pipreqs import pipreqs
 class TestPipreqs(unittest.TestCase):
 
     def setUp(self):
-        self.modules = ['flask', 'requests', 'sqlalchemy',
+        self.modules = [
-                        'docopt', 'boto', 'ipython', 'pyflakes', 'nose',
+            'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython',
-                        'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ]
+            'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee',
+            'ujson', 'nonexistendmodule', 'bs4',
+            'after_method_is_valid_even_if_not_pep8'
+            ]
         self.modules2 = ['beautifulsoup4']
         self.local = ["docopt", "requests", "nose", 'pyflakes']
         self.project = os.path.join(os.path.dirname(__file__), "_data")
-        self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data")
+        self.project_clean = os.path.join(
-        self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore")
+            os.path.dirname(__file__),
-        self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps")
+            "_data_clean"
+            )
+        self.project_invalid = os.path.join(
+            os.path.dirname(__file__),
+            "_invalid_data"
+            )
+        self.project_with_ignore_directory = os.path.join(
+            os.path.dirname(__file__),
+            "_data_ignore"
+            )
+        self.project_with_duplicated_deps = os.path.join(
+            os.path.dirname(__file__),
+            "_data_duplicated_deps"
+            )
         self.requirements_path = os.path.join(self.project, "requirements.txt")
         self.alt_requirement_path = os.path.join(
-            self.project, "requirements2.txt")
+            self.project,
+            "requirements2.txt"
+            )
 
     def test_get_all_imports(self):
         imports = pipreqs.get_all_imports(self.project)
-        self.assertEqual(len(imports), 13)
+        self.assertEqual(len(imports), 15)
         for item in imports:
             self.assertTrue(
                 item.lower() in self.modules, "Import is missing: " + item)
@@ -54,7 +72,8 @@ class TestPipreqs(unittest.TestCase):
         """
         Test that invalid python files cannot be imported.
         """
-        self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
+        self.assertRaises(
+            SyntaxError, pipreqs.get_all_imports, self.project_invalid)
 
     def test_get_imports_info(self):
         """
@@ -62,8 +81,9 @@ class TestPipreqs(unittest.TestCase):
         """
         imports = pipreqs.get_all_imports(self.project)
         with_info = pipreqs.get_imports_info(imports)
-        # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8"
+        # Should contain 10 items without the "nonexistendmodule" and
-        self.assertEqual(len(with_info), 11)
+        # "after_method_is_valid_even_if_not_pep8"
+        self.assertEqual(len(with_info), 13)
         for item in with_info:
             self.assertTrue(
                 item['name'].lower() in self.modules,
@@ -77,10 +97,12 @@ class TestPipreqs(unittest.TestCase):
 
     def test_get_use_local_only(self):
         """
-        Test without checking PyPI, check to see if names of local imports matches what we expect
+        Test without checking PyPI, check to see if names of local
+        imports matches what we expect
 
         - Note even though pyflakes isn't in requirements.txt,
-          It's added to locals since it is a development dependency for testing
+          It's added to locals since it is a development dependency
+          for testing
         """
         # should find only docopt and requests
         imports_with_info = pipreqs.get_import_local(self.modules)
@@ -89,24 +111,28 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init(self):
         """
-        Test that all modules we will test upon, are in requirements file
+        Test that all modules we will test upon are in requirements file
         """
         pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
                       '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None,
-                      '--diff': None, '--clean': None})
+                      '--diff': None, '--clean': None, '--mode': None})
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.read().lower()
             for item in self.modules[:-3]:
                 self.assertTrue(item.lower() in data)
+        # It should be sorted based on names.
+        data = data.strip().split('\n')
+        self.assertEqual(data, sorted(data))
 
     def test_init_local_only(self):
         """
-        Test that items listed in requirements.text are the same as locals expected
+        Test that items listed in requirements.text are the same
+        as locals expected
         """
         pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
                       '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None,
-                      '--diff': None, '--clean': None})
+                      '--diff': None, '--clean': None, '--mode': None})
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.readlines()
@@ -116,11 +142,12 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init_savepath(self):
         """
-        Test that we can save requiremnts.tt correctly to a different path
+        Test that we can save requirements.txt correctly
+        to a different path
         """
-        pipreqs.init({'<path>': self.project, '--savepath':
+        pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path, 
-                      self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None,  '--print': False,
+                      '--use-local': None, '--proxy':None, '--pypi-server':None,  '--print': False,
-                      "--diff": None, "--clean": None})
+                      '--diff': None, '--clean': None, '--mode': None})
         assert os.path.exists(self.alt_requirement_path) == 1
         with open(self.alt_requirement_path, "r") as f:
             data = f.read().lower()
@@ -131,13 +158,14 @@ class TestPipreqs(unittest.TestCase):
 
     def test_init_overwrite(self):
         """
-        Test that if requiremnts.txt exists, it will not automatically be overwritten
+        Test that if requiremnts.txt exists, it will not be
+        automatically overwritten
         """
         with open(self.requirements_path, "w") as f:
             f.write("should_not_be_overwritten")
-        pipreqs.init({'<path>': self.project, '--savepath': None,
+        pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None, 
-                      '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
+                      '--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
-                      "--diff": None, "--clean": None})
+                      '--diff': None, '--clean': None, '--mode': None})
         assert os.path.exists(self.requirements_path) == 1
         with open(self.requirements_path, "r") as f:
             data = f.read().lower()
@@ -145,35 +173,43 @@ class TestPipreqs(unittest.TestCase):
 
     def test_get_import_name_without_alias(self):
         """
-        Test that function get_name_without_alias() will work on a string.
+        Test that function get_name_without_alias()
-        - Note: This isn't truly needed when pipreqs is walking the AST to find imports
+        will work on a string.
+        - Note: This isn't truly needed when pipreqs is walking
+          the AST to find imports
         """
         import_name_with_alias = "requests as R"
         expected_import_name_without_alias = "requests"
         import_name_without_aliases = pipreqs.get_name_without_alias(
             import_name_with_alias)
         self.assertEqual(
-            import_name_without_aliases, expected_import_name_without_alias)
+            import_name_without_aliases,
+            expected_import_name_without_alias
+            )
 
     def test_custom_pypi_server(self):
         """
         Test that trying to get a custom pypi sever fails correctly
         """
-        self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'<path>': self.project, '--savepath': None, '--print': False,
+        self.assertRaises(
-                          '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'})
+            requests.exceptions.MissingSchema, pipreqs.init,
+            {'<path>': self.project, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True, '--proxy': None,
+             '--pypi-server': 'nonexistent'}
+            )
 
     def test_ignored_directory(self):
         """
         Test --ignore parameter
         """
         pipreqs.init(
-            {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
+            {'<path>': self.project_with_ignore_directory, '--savepath': None, 
-                      '--use-local': None, '--force': True,
+                      '--print': False, '--use-local': None, '--force': True,
-                      '--proxy':None,
+                      '--proxy':None, '--pypi-server':None,
-                      '--pypi-server':None,
                       '--ignore':'.ignored_dir,.ignore_second',
                       '--diff': None,
-                      '--clean': None
+                      '--clean': None,
+                      '--mode': None
              }
         )
         with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
@@ -181,9 +217,27 @@ class TestPipreqs(unittest.TestCase):
             for item in ['click', 'getpass']:
                 self.assertFalse(item.lower() in data)
 
-    def test_omit_version(self):
+    def test_dynamic_version_no_pin_scheme(self):
         """
-        Test --no-pin parameter
+        Test --mode=no-pin
+        """
+        pipreqs.init(
+            {'<path>': self.project_with_ignore_directory, '--savepath': None, 
+             '--print': False, '--use-local': None, '--force': True,
+             '--proxy': None, '--pypi-server': None,
+             '--diff': None,
+             '--clean': None,
+             '--mode': 'no-pin'
+             }
+        )
+        with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
+            data = f.read().lower()
+            for item in ['beautifulsoup4', 'boto']:
+                self.assertTrue(item.lower() in data)
+
+    def test_dynamic_version_gt_scheme(self):
+        """
+        Test --mode=gt
         """
         pipreqs.init(
             {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
@@ -192,13 +246,81 @@ class TestPipreqs(unittest.TestCase):
              '--pypi-server': None,
              '--diff': None,
              '--clean': None,
-             '--no-pin': True
+             '--mode': 'gt'
              }
         )
         with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
+            data = f.readlines()
+            for item in data:
+                symbol = '>='
+                message = 'symbol is not in item'
+                self.assertIn(symbol, item, message)
+
+    def test_dynamic_version_compat_scheme(self):
+        """
+        Test --mode=compat
+        """
+        pipreqs.init(
+            {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True,
+             '--proxy': None,
+             '--pypi-server': None,
+             '--diff': None,
+             '--clean': None,
+             '--mode': 'compat'
+             }
+        )
+        with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
+            data = f.readlines()
+            for item in data:
+                symbol = '~='
+                message = 'symbol is not in item'
+                self.assertIn(symbol, item, message)
+
+    def test_clean(self):
+        """
+        Test --clean parameter
+        """
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True, '--proxy': None,
+             '--pypi-server': None, '--diff': None, '--clean': None,
+             '--mode': None}
+            )
+        assert os.path.exists(self.requirements_path) == 1
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+            '--use-local': None, '--force': None, '--proxy': None,
+            '--pypi-server': None, '--diff': None,
+            '--clean': self.requirements_path, '--mode': 'non-pin'}
+            )
+        with open(self.requirements_path, "r") as f:
             data = f.read().lower()
-            for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']:
+            for item in self.modules[:-3]:
-                self.assertFalse(item.lower() in data)
+                self.assertTrue(item.lower() in data)
+
+    def test_clean_with_imports_to_clean(self):
+        """
+        Test --clean parameter when there are imports to clean
+        """
+        cleaned_module = 'sqlalchemy'
+        pipreqs.init(
+            {'<path>': self.project, '--savepath': None, '--print': False,
+            '--use-local': None, '--force': True, '--proxy': None,
+            '--pypi-server': None, '--diff': None, '--clean': None,
+            '--mode': None}
+            )
+        assert os.path.exists(self.requirements_path) == 1
+        modules_clean = [m for m in self.modules if m != cleaned_module]
+        pipreqs.init(
+            {'<path>': self.project_clean, '--savepath': None,
+            '--print': False, '--use-local': None, '--force': None,
+            '--proxy': None, '--pypi-server': None, '--diff': None,
+            '--clean': self.requirements_path, '--mode': 'non-pin'}
+            )
+        with open(self.requirements_path, "r") as f:
+            data = f.read().lower()
+            self.assertTrue(cleaned_module not in data)
 
     def tearDown(self):
         """

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py34, py35, py36, pypy, flake8
+envlist = py27, py34, py35, py36, pypy3, flake8
 
 [testenv]
 setenv =