Merge branch 'master' into mysql-python

README.rst

@@ -49,6 +49,7 @@ Usage
         --force               Overwrite existing requirements.txt
         --diff <file>         Compare modules in requirements.txt to project imports.
         --clean <file>        Clean up requirements.txt by removing modules that are not imported in project.
+        --no-pin              Omit version of output packages.
 Example
 -------
 
@@ -69,5 +70,5 @@ Why not pip freeze?
 -------------------
 
 - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. 
-- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv)
+- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``).
-- and sometimes you just need to create requirements.txt for a new project without installing modules.
+- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules.

pipreqs/mapping

@@ -1,3 +1,4 @@
+AFQ:pyAFQ
 AG_fft_tools:agpy
 ANSI:pexpect
 Adafruit:Adafruit_Libraries
@@ -13,6 +14,7 @@ Crypto:pycryptodome
 Cryptodome:pycryptodomex
 FSM:pexpect
 FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper
+functional:pyfunctional
 GeoBaseMain:GeoBasesDev
 GeoBases:GeoBasesDev
 Globals:Zope2
@@ -22,6 +24,7 @@ Kittens:astro_kittens
 Levenshtein:python_Levenshtein
 Lifetime:Zope2
 MethodObject:ExtensionClass
+MySQLdb:MySQL-python
 OFS:Zope2
 OpenGL:PyOpenGL
 OpenSSL:pyOpenSSL
@@ -536,6 +539,7 @@ cassandra:cassandra_driver
 cassandralauncher:CassandraLauncher
 cc42:42qucc
 cerberus:Cerberus
+cfnlint:cfn-lint
 chameleon:Chameleon
 charmtools:charm_tools
 chef:PyChef
@@ -580,16 +584,18 @@ daemon:python_daemon
 dare:DARE
 dateutil:python_dateutil
 dawg:DAWG
-decouple:python-decouple
 deb822:python_debian
 debian:python_debian
+decouple:python-decouple
 demo:webunit
+demosongs:PySynth
 deployer:juju_deployer
 depot:filedepot
 devtools:tg.devtools
 dgis:2gis
 dhtmlparser:pyDHTMLParser
 digitalocean:python_digitalocean
+discord:discord.py
 distribute_setup:ez_setup
 distutils2:Distutils2
 django:Django
@@ -699,7 +705,6 @@ html:pies2overrides
 htmloutput:nosehtmloutput
 http:pies2overrides
 hvad:django_hvad
-krbv:krbV
 i99fix:199Fix
 igraph:python_igraph
 imdb:IMDbPY
@@ -725,6 +730,7 @@ keyczar:python_keyczar
 keyedcache:django_keyedcache
 keystoneclient:python_keystoneclient
 kickstarter:kickstart
+krbv:krbV
 kss:kss.core
 kuyruk:Kuyruk
 langconv:AdvancedLangConv
@@ -773,6 +779,8 @@ mimeparse:python_mimeparse
 minitage:minitage.paste
 minitage:minitage.recipe.common
 missingdrawables:android_missingdrawables
+mixfiles:PySynth
+mkfreq:PySynth
 mkrst_themes:2lazy2rest
 mockredis:mockredispy
 modargs:python_modargs
@@ -788,12 +796,12 @@ monthdelta:MonthDelta
 mopidy:Mopidy
 mopytools:MoPyTools
 mptt:django_mptt
+mpv:python-mpv
 mrbob:mr.bob
 msgpack:msgpack_python
 mutations:aino_mutations
 mws:amazon_mws
 mysql:mysql_connector_repackaged
-MySQLdb:MySQL-python
 native_tags:django_native_tags
 ndg:ndg_httpsclient
 nereid:trytond_nereid
@@ -824,12 +832,14 @@ past:future
 paste:PasteScript
 path:forked_path
 path:path.py
+patricia:patricia-trie
 paver:Paver
 peak:ProxyTypes
 picasso:anderson.picasso
 picklefield:django-picklefield
 pilot:BigJob
 pivotal:pivotal_py
+play_wav:PySynth
 playhouse:peewee
 plivoxml:plivo
 plone:plone.alterego
@@ -913,7 +923,6 @@ plone:plone.z3cform
 plonetheme:plonetheme.barceloneta
 png:pypng
 polymorphic:django_polymorphic
-portalocker:ConcurrentLogHandler
 postmark:python_postmark
 powerprompt:bash_powerprompt
 prefetch:django-prefetch
@@ -951,6 +960,14 @@ pyrimaa:AEI
 pysideuic:PySide
 pysqlite2:adhocracy_pysqlite
 pysqlite2:pysqlite
+pysynth_b:PySynth
+pysynth_beeper:PySynth
+pysynth_c:PySynth
+pysynth_d:PySynth
+pysynth_e:PySynth
+pysynth_p:PySynth
+pysynth_s:PySynth
+pysynth_samp:PySynth
 pythongettext:python_gettext
 pythonjsonlogger:python_json_logger
 pyutilib:PyUtilib
@@ -985,6 +1002,7 @@ ruamel:ruamel.base
 s2repoze:pysaml2
 saga:saga_python
 saml2:pysaml2
+samtranslator:aws-sam-translator
 sass:libsass
 sassc:libsass
 sasstests:libsass
@@ -1008,6 +1026,7 @@ singleton:pysingleton
 sittercommon:cerebrod
 skbio:scikit_bio
 sklearn:scikit_learn
+slack:slackclient
 slugify:unicode_slugify
 smarkets:smk_python_sdk
 snappy:ctypes_snappy

pipreqs/pipreqs.py

@@ -31,6 +31,7 @@ Options:
                           imports.
     --clean <file>        Clean up requirements.txt by removing modules
                           that are not imported in project.
+    --no-pin              Omit version of output packages.
 """
 from __future__ import print_function, absolute_import
 from contextlib import contextmanager
@@ -156,8 +157,8 @@ def get_all_imports(
     return list(packages - data)
 
 
-def filter_line(l):
+def filter_line(line):
-    return len(l) > 0 and l[0] != "#"
+    return len(line) > 0 and line[0] != "#"
 
 
 def generate_requirements_file(path, imports):
@@ -309,9 +310,10 @@ def parse_requirements(file_):
         logging.error("Failed on file: {}".format(file_))
         raise
     else:
-        data = [x.strip() for x in f.readlines() if x != "\n"]
+        try:
-    finally:
+            data = [x.strip() for x in f.readlines() if x != "\n"]
-        f.close()
+        finally:
+            f.close()
 
     data = [x for x in data if x[0].isalpha()]
 
@@ -375,16 +377,17 @@ def clean(file_, imports):
         logging.error("Failed on file: {}".format(file_))
         raise
     else:
-        for i in f.readlines():
+        try:
-            if re_remove.match(i) is None:
+            for i in f.readlines():
-                to_write.append(i)
+                if re_remove.match(i) is None:
-        f.seek(0)
+                    to_write.append(i)
-        f.truncate()
+            f.seek(0)
+            f.truncate()
 
-        for i in to_write:
+            for i in to_write:
-            f.write(i)
+                f.write(i)
-    finally:
+        finally:
-        f.close()
+            f.close()
 
     logging.info("Successfully cleaned up requirements in " + file_)
 
@@ -427,6 +430,8 @@ def init(args):
         imports = local + get_imports_info(difference,
                                            proxy=proxy,
                                            pypi_server=pypi_server)
+    # sort imports based on lowercase name of package, similar to `pip freeze`.
+    imports = sorted(imports, key=lambda x: x['name'].lower())
 
     path = (args["--savepath"] if args["--savepath"] else
             os.path.join(input_path, "requirements.txt"))
@@ -447,6 +452,9 @@ def init(args):
                         "use --force to overwrite it")
         return
 
+    if args.get('--no-pin'):
+        imports = [{'name': item["name"], 'version': ''} for item in imports]
+
     if args["--print"]:
         output_requirements(imports)
         logging.info("Successfully output requirements")

pipreqs/stdlib

@@ -1,6 +1,8 @@
 __builtin__
 __future__
 __main__
+_dummy_thread
+_thread
 _winreg
 abc
 aepack
@@ -15,6 +17,7 @@ argparse
 array
 ast
 asynchat
+asyncio
 asyncore
 atexit
 audioop
@@ -28,6 +31,7 @@ binhex
 bisect
 bsddb
 buildtools
+builtins
 bz2
 calendar
 Carbon
@@ -101,6 +105,7 @@ code
 codecs
 codeop
 collections
+collections.abc
 ColorPicker
 colorsys
 commands
@@ -108,12 +113,16 @@ compileall
 compiler
 compiler.ast
 compiler.visitor
+concurrent
+concurrent.futures
 ConfigParser
+configparser
 contextlib
 Cookie
 cookielib
 copy
 copy_reg
+copyreg
 cPickle
 cProfile
 crypt
@@ -127,6 +136,9 @@ curses.textpad
 datetime
 dbhash
 dbm
+dbm.dumb
+dbm.gnu
+dbm.ndbm
 decimal
 DEVICE
 difflib
@@ -188,21 +200,27 @@ dummy_threading
 EasyDialogs
 email
 email.charset
+email.contentmanager
 email.encoders
 email.errors
 email.generator
 email.header
+email.headerregistry
 email.iterators
 email.message
 email.mime
 email.parser
+email.policy
 email.utils
 encodings
 encodings.idna
+encodings.mbcs
 encodings.utf_8_sig
 ensurepip
+enum
 errno
 exceptions
+faulthandler
 fcntl
 filecmp
 fileinput
@@ -226,8 +244,8 @@ gensuitemodule
 getopt
 getpass
 gettext
-gl
 GL
+gl
 glob
 grp
 gzip
@@ -236,9 +254,17 @@ heapq
 hmac
 hotshot
 hotshot.stats
+html
+html.entities
+html.parser
 htmlentitydefs
 htmllib
 HTMLParser
+http
+http.client
+http.cookiejar
+http.cookies
+http.server
 httplib
 ic
 icopen
@@ -248,12 +274,17 @@ imgfile
 imghdr
 imp
 importlib
+importlib.abc
+importlib.machinery
+importlib.util
 imputil
 inspect
 io
+ipaddress
 itertools
 jpeg
 json
+json.tool
 keyword
 lib2to3
 linecache
@@ -261,6 +292,7 @@ locale
 logging
 logging.config
 logging.handlers
+lzma
 macerrors
 MacOS
 macostools
@@ -301,11 +333,13 @@ os
 os.path
 ossaudiodev
 parser
+pathlib
 pdb
 pickle
 pickletools
 pipes
 PixMapWrapper
+pkg_resources
 pkgutil
 platform
 plistlib
@@ -322,10 +356,12 @@ py_compile
 pyclbr
 pydoc
 Queue
+queue
 quopri
 random
 re
 readline
+reprlib
 resource
 rexec
 rfc822
@@ -335,7 +371,9 @@ runpy
 sched
 ScrolledText
 select
+selectors
 sets
+setuptools
 sgmllib
 sha
 shelve
@@ -350,10 +388,12 @@ smtplib
 sndhdr
 socket
 SocketServer
+socketserver
 spwd
 sqlite3
 ssl
 stat
+statistics
 statvfs
 string
 StringIO
@@ -361,8 +401,8 @@ stringprep
 struct
 subprocess
 sunau
-sunaudiodev
 SUNAUDIODEV
+sunaudiodev
 symbol
 symtable
 sys
@@ -374,6 +414,7 @@ telnetlib
 tempfile
 termios
 test
+test.support
 test.test_support
 textwrap
 thread
@@ -382,17 +423,30 @@ time
 timeit
 Tix
 Tkinter
+tkinter
+tkinter.scrolledtext
+tkinter.tix
+tkinter.ttk
 token
 tokenize
 trace
 traceback
+tracemalloc
 ttk
 tty
 turtle
+turtledemo
 types
+typing
 unicodedata
 unittest
+unittest.mock
 urllib
+urllib.error
+urllib.parse
+urllib.request
+urllib.response
+urllib.robotparser
 urllib2
 urlparse
 user
@@ -401,6 +455,7 @@ UserList
 UserString
 uu
 uuid
+venv
 videoreader
 W
 warnings
@@ -408,6 +463,7 @@ wave
 weakref
 webbrowser
 whichdb
+winreg
 winsound
 wsgiref
 wsgiref.handlers
@@ -422,654 +478,17 @@ xml.dom.minidom
 xml.dom.pulldom
 xml.etree.ElementTree
 xml.parsers.expat
+xml.parsers.expat.errors
+xml.parsers.expat.model
 xml.sax
 xml.sax.handler
 xml.sax.saxutils
 xml.sax.xmlreader
+xmlrpc
+xmlrpc.client
+xmlrpc.server
 xmlrpclib
-zipfile
+yp
-zipimport
-zlib
-__future__
-__main__
-_dummy_thread
-_thread
-abc
-aifc
-argparse
-array
-ast
-asynchat
-asyncio
-asyncore
-atexit
-audioop
-base64
-bdb
-binascii
-binhex
-bisect
-builtins
-bz2
-calendar
-cgi
-cgitb
-chunk
-cmath
-cmd
-code
-codecs
-codeop
-collections
-collections.abc
-colorsys
-compileall
-concurrent
-concurrent.futures
-configparser
-contextlib
-copy
-copyreg
-cProfile
-crypt
-csv
-ctypes
-curses
-curses.ascii
-curses.panel
-curses.textpad
-datetime
-dbm
-dbm.dumb
-dbm.gnu
-dbm.ndbm
-decimal
-difflib
-dis
-distutils
-distutils.archive_util
-distutils.bcppcompiler
-distutils.ccompiler
-distutils.cmd
-distutils.command
-distutils.command.bdist
-distutils.command.bdist_dumb
-distutils.command.bdist_msi
-distutils.command.bdist_packager
-distutils.command.bdist_rpm
-distutils.command.bdist_wininst
-distutils.command.build
-distutils.command.build_clib
-distutils.command.build_ext
-distutils.command.build_py
-distutils.command.build_scripts
-distutils.command.check
-distutils.command.clean
-distutils.command.config
-distutils.command.install
-distutils.command.install_data
-distutils.command.install_headers
-distutils.command.install_lib
-distutils.command.install_scripts
-distutils.command.register
-distutils.command.sdist
-distutils.core
-distutils.cygwinccompiler
-distutils.debug
-distutils.dep_util
-distutils.dir_util
-distutils.dist
-distutils.errors
-distutils.extension
-distutils.fancy_getopt
-distutils.file_util
-distutils.filelist
-distutils.log
-distutils.msvccompiler
-distutils.spawn
-distutils.sysconfig
-distutils.text_file
-distutils.unixccompiler
-distutils.util
-distutils.version
-doctest
-dummy_threading
-email
-email.charset
-email.contentmanager
-email.encoders
-email.errors
-email.generator
-email.header
-email.headerregistry
-email.iterators
-email.message
-email.mime
-email.parser
-email.policy
-email.utils
-encodings
-encodings.idna
-encodings.mbcs
-encodings.utf_8_sig
-ensurepip
-enum
-errno
-faulthandler
-fcntl
-filecmp
-fileinput
-fnmatch
-formatter
-fpectl
-fractions
-ftplib
-functools
-gc
-getopt
-getpass
-gettext
-glob
-grp
-gzip
-hashlib
-heapq
-hmac
-html
-html.entities
-html.parser
-http
-http.client
-http.cookiejar
-http.cookies
-http.server
-imaplib
-imghdr
-imp
-importlib
-importlib.abc
-importlib.machinery
-importlib.util
-inspect
-io
-ipaddress
-itertools
-json
-keyword
-lib2to3
-linecache
-locale
-logging
-logging.config
-logging.handlers
-lzma
-macpath
-mailbox
-mailcap
-marshal
-math
-mimetypes
-mmap
-modulefinder
-msilib
-msvcrt
-multiprocessing
-multiprocessing.connection
-multiprocessing.dummy
-multiprocessing.managers
-multiprocessing.pool
-multiprocessing.sharedctypes
-netrc
-nis
-nntplib
-numbers
-operator
-optparse
-os
-os.path
-ossaudiodev
-parser
-pathlib
-pdb
-pickle
-pickletools
-pipes
-pkgutil
-platform
-plistlib
-poplib
-posix
-pprint
-profile
-pstats
-pty
-pwd
-py_compile
-pyclbr
-pydoc
-queue
-quopri
-random
-re
-readline
-reprlib
-resource
-rlcompleter
-runpy
-sched
-select
-selectors
-shelve
-shlex
-shutil
-signal
-site
-smtpd
-smtplib
-sndhdr
-socket
-socketserver
-spwd
-sqlite3
-ssl
-stat
-statistics
-string
-stringprep
-struct
-subprocess
-sunau
-symbol
-symtable
-sys
-sysconfig
-syslog
-tabnanny
-tarfile
-telnetlib
-tempfile
-termios
-test
-test.support
-textwrap
-threading
-time
-timeit
-tkinter
-tkinter.scrolledtext
-tkinter.tix
-tkinter.ttk
-token
-tokenize
-trace
-traceback
-tracemalloc
-tty
-turtle
-turtledemo
-types
-unicodedata
-unittest
-unittest.mock
-urllib
-urllib.error
-urllib.parse
-urllib.request
-urllib.response
-urllib.robotparser
-uu
-uuid
-venv
-warnings
-wave
-weakref
-webbrowser
-winreg
-winsound
-wsgiref
-wsgiref.handlers
-wsgiref.headers
-wsgiref.simple_server
-wsgiref.util
-wsgiref.validate
-xdrlib
-xml
-xml.dom
-xml.dom.minidom
-xml.dom.pulldom
-xml.etree.ElementTree
-xml.parsers.expat
-xml.parsers.expat.errors
-xml.parsers.expat.model
-xml.sax
-xml.sax.handler
-xml.sax.saxutils
-xml.sax.xmlreader
-xmlrpc
-xmlrpc.client
-xmlrpc.server
-zipfile
-zipimport
-zlib
-__future__
-__main__
-_dummy_thread
-_thread
-abc
-aifc
-argparse
-array
-ast
-asynchat
-asyncio
-asyncore
-atexit
-audioop
-base64
-bdb
-binascii
-binhex
-bisect
-builtins
-bz2
-calendar
-cgi
-cgitb
-chunk
-cmath
-cmd
-code
-codecs
-codeop
-collections
-collections.abc
-colorsys
-compileall
-concurrent
-concurrent.futures
-configparser
-contextlib
-copy
-copyreg
-cProfile
-crypt
-csv
-ctypes
-curses
-curses.ascii
-curses.panel
-curses.textpad
-datetime
-dbm
-dbm.dumb
-dbm.gnu
-dbm.ndbm
-decimal
-difflib
-dis
-distutils
-distutils.archive_util
-distutils.bcppcompiler
-distutils.ccompiler
-distutils.cmd
-distutils.command
-distutils.command.bdist
-distutils.command.bdist_dumb
-distutils.command.bdist_msi
-distutils.command.bdist_packager
-distutils.command.bdist_rpm
-distutils.command.bdist_wininst
-distutils.command.build
-distutils.command.build_clib
-distutils.command.build_ext
-distutils.command.build_py
-distutils.command.build_scripts
-distutils.command.check
-distutils.command.clean
-distutils.command.config
-distutils.command.install
-distutils.command.install_data
-distutils.command.install_headers
-distutils.command.install_lib
-distutils.command.install_scripts
-distutils.command.register
-distutils.command.sdist
-distutils.core
-distutils.cygwinccompiler
-distutils.debug
-distutils.dep_util
-distutils.dir_util
-distutils.dist
-distutils.errors
-distutils.extension
-distutils.fancy_getopt
-distutils.file_util
-distutils.filelist
-distutils.log
-distutils.msvccompiler
-distutils.spawn
-distutils.sysconfig
-distutils.text_file
-distutils.unixccompiler
-distutils.util
-distutils.version
-doctest
-dummy_threading
-email
-email.charset
-email.contentmanager
-email.encoders
-email.errors
-email.generator
-email.header
-email.headerregistry
-email.iterators
-email.message
-email.mime
-email.parser
-email.policy
-email.utils
-encodings
-encodings.idna
-encodings.mbcs
-encodings.utf_8_sig
-ensurepip
-enum
-errno
-faulthandler
-fcntl
-filecmp
-fileinput
-fnmatch
-formatter
-fpectl
-fractions
-ftplib
-functools
-gc
-getopt
-getpass
-gettext
-glob
-grp
-gzip
-hashlib
-heapq
-hmac
-html
-html.entities
-html.parser
-http
-http.client
-http.cookiejar
-http.cookies
-http.server
-imaplib
-imghdr
-imp
-importlib
-importlib.abc
-importlib.machinery
-importlib.util
-inspect
-io
-ipaddress
-itertools
-json
-json.tool
-keyword
-lib2to3
-linecache
-locale
-logging
-logging.config
-logging.handlers
-lzma
-macpath
-mailbox
-mailcap
-marshal
-math
-mimetypes
-mmap
-modulefinder
-msilib
-msvcrt
-multiprocessing
-multiprocessing.connection
-multiprocessing.dummy
-multiprocessing.managers
-multiprocessing.pool
-multiprocessing.sharedctypes
-netrc
-nis
-nntplib
-numbers
-operator
-optparse
-os
-os.path
-ossaudiodev
-parser
-pathlib
-pdb
-pickle
-pickletools
-pipes
-pkgutil
-platform
-plistlib
-poplib
-posix
-pprint
-profile
-pstats
-pty
-pwd
-py_compile
-pyclbr
-pydoc
-queue
-quopri
-random
-re
-readline
-reprlib
-resource
-rlcompleter
-runpy
-sched
-select
-selectors
-shelve
-shlex
-shutil
-signal
-site
-smtpd
-smtplib
-sndhdr
-socket
-socketserver
-spwd
-sqlite3
-ssl
-stat
-statistics
-string
-stringprep
-struct
-subprocess
-sunau
-symbol
-symtable
-sys
-sysconfig
-syslog
-tabnanny
-tarfile
-telnetlib
-tempfile
-termios
-test
-test.support
-textwrap
-threading
-time
-timeit
-tkinter
-tkinter.scrolledtext
-tkinter.tix
-tkinter.ttk
-token
-tokenize
-trace
-traceback
-tracemalloc
-tty
-turtle
-turtledemo
-types
-unicodedata
-unittest
-unittest.mock
-urllib
-urllib.error
-urllib.parse
-urllib.request
-urllib.response
-urllib.robotparser
-uu
-uuid
-venv
-warnings
-wave
-weakref
-webbrowser
-winreg
-winsound
-wsgiref
-wsgiref.handlers
-wsgiref.headers
-wsgiref.simple_server
-wsgiref.util
-wsgiref.validate
-xdrlib
-xml
-xml.dom
-xml.dom.minidom
-xml.dom.pulldom
-xml.etree.ElementTree
-xml.parsers.expat
-xml.parsers.expat.errors
-xml.parsers.expat.model
-xml.sax
-xml.sax.handler
-xml.sax.saxutils
-xml.sax.xmlreader
-xmlrpc
-xmlrpc.client
-xmlrpc.server
 zipapp
 zipfile
 zipimport

tests/test_pipreqs.py

@@ -99,6 +99,9 @@ class TestPipreqs(unittest.TestCase):
             data = f.read().lower()
             for item in self.modules[:-3]:
                 self.assertTrue(item.lower() in data)
+        # It should be sorted based on names.
+        data = data.strip().split('\n')
+        self.assertEqual(data, sorted(data))
 
     def test_init_local_only(self):
         """
@@ -181,6 +184,24 @@ class TestPipreqs(unittest.TestCase):
             for item in ['click', 'getpass']:
                 self.assertFalse(item.lower() in data)
 
+    def test_omit_version(self):
+        """
+        Test --no-pin parameter
+        """
+        pipreqs.init(
+            {'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
+             '--use-local': None, '--force': True,
+             '--proxy': None,
+             '--pypi-server': None,
+             '--diff': None,
+             '--clean': None,
+             '--no-pin': True
+             }
+        )
+        with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
+            data = f.read().lower()
+            for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']:
+                self.assertFalse(item.lower() in data)
 
     def tearDown(self):
         """