diff --git a/README.rst b/README.rst index b6cbd30..2a33b60 100644 --- a/README.rst +++ b/README.rst @@ -49,6 +49,7 @@ Usage --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. --clean Clean up requirements.txt by removing modules that are not imported in project. + --no-pin Omit version of output packages. Example ------- @@ -69,5 +70,5 @@ Why not pip freeze? ------------------- - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. -- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv) -- and sometimes you just need to create requirements.txt for a new project without installing modules. +- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``). +- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules. diff --git a/pipreqs/mapping b/pipreqs/mapping index 06fa00c..cf90aa5 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -1,3 +1,4 @@ +AFQ:pyAFQ AG_fft_tools:agpy ANSI:pexpect Adafruit:Adafruit_Libraries @@ -13,6 +14,7 @@ Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper +functional:pyfunctional GeoBaseMain:GeoBasesDev GeoBases:GeoBasesDev Globals:Zope2 @@ -22,6 +24,7 @@ Kittens:astro_kittens Levenshtein:python_Levenshtein Lifetime:Zope2 MethodObject:ExtensionClass +MySQLdb:MySQL-python OFS:Zope2 OpenGL:PyOpenGL OpenSSL:pyOpenSSL @@ -536,6 +539,7 @@ cassandra:cassandra_driver cassandralauncher:CassandraLauncher cc42:42qucc cerberus:Cerberus +cfnlint:cfn-lint chameleon:Chameleon charmtools:charm_tools chef:PyChef @@ -580,16 +584,18 @@ daemon:python_daemon dare:DARE dateutil:python_dateutil dawg:DAWG -decouple:python-decouple deb822:python_debian debian:python_debian +decouple:python-decouple demo:webunit +demosongs:PySynth deployer:juju_deployer depot:filedepot devtools:tg.devtools dgis:2gis dhtmlparser:pyDHTMLParser digitalocean:python_digitalocean +discord:discord.py distribute_setup:ez_setup distutils2:Distutils2 django:Django @@ -699,7 +705,6 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad -krbv:krbV i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -725,6 +730,7 @@ keyczar:python_keyczar keyedcache:django_keyedcache keystoneclient:python_keystoneclient kickstarter:kickstart +krbv:krbV kss:kss.core kuyruk:Kuyruk langconv:AdvancedLangConv @@ -773,6 +779,8 @@ mimeparse:python_mimeparse minitage:minitage.paste minitage:minitage.recipe.common missingdrawables:android_missingdrawables +mixfiles:PySynth +mkfreq:PySynth mkrst_themes:2lazy2rest mockredis:mockredispy modargs:python_modargs @@ -788,12 +796,12 @@ monthdelta:MonthDelta mopidy:Mopidy mopytools:MoPyTools mptt:django_mptt +mpv:python-mpv mrbob:mr.bob msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged -MySQLdb:MySQL-python native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -824,12 +832,14 @@ past:future paste:PasteScript path:forked_path path:path.py +patricia:patricia-trie paver:Paver peak:ProxyTypes picasso:anderson.picasso picklefield:django-picklefield pilot:BigJob pivotal:pivotal_py +play_wav:PySynth playhouse:peewee plivoxml:plivo plone:plone.alterego @@ -913,7 +923,6 @@ plone:plone.z3cform plonetheme:plonetheme.barceloneta png:pypng polymorphic:django_polymorphic -portalocker:ConcurrentLogHandler postmark:python_postmark powerprompt:bash_powerprompt prefetch:django-prefetch @@ -951,6 +960,14 @@ pyrimaa:AEI pysideuic:PySide pysqlite2:adhocracy_pysqlite pysqlite2:pysqlite +pysynth_b:PySynth +pysynth_beeper:PySynth +pysynth_c:PySynth +pysynth_d:PySynth +pysynth_e:PySynth +pysynth_p:PySynth +pysynth_s:PySynth +pysynth_samp:PySynth pythongettext:python_gettext pythonjsonlogger:python_json_logger pyutilib:PyUtilib @@ -985,6 +1002,7 @@ ruamel:ruamel.base s2repoze:pysaml2 saga:saga_python saml2:pysaml2 +samtranslator:aws-sam-translator sass:libsass sassc:libsass sasstests:libsass @@ -1008,6 +1026,7 @@ singleton:pysingleton sittercommon:cerebrod skbio:scikit_bio sklearn:scikit_learn +slack:slackclient slugify:unicode_slugify smarkets:smk_python_sdk snappy:ctypes_snappy diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 4e7b39f..42a4e45 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,6 +31,7 @@ Options: imports. --clean Clean up requirements.txt by removing modules that are not imported in project. + --no-pin Omit version of output packages. """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -156,8 +157,8 @@ def get_all_imports( return list(packages - data) -def filter_line(l): - return len(l) > 0 and l[0] != "#" +def filter_line(line): + return len(line) > 0 and line[0] != "#" def generate_requirements_file(path, imports): @@ -309,9 +310,10 @@ def parse_requirements(file_): logging.error("Failed on file: {}".format(file_)) raise else: - data = [x.strip() for x in f.readlines() if x != "\n"] - finally: - f.close() + try: + data = [x.strip() for x in f.readlines() if x != "\n"] + finally: + f.close() data = [x for x in data if x[0].isalpha()] @@ -375,16 +377,17 @@ def clean(file_, imports): logging.error("Failed on file: {}".format(file_)) raise else: - for i in f.readlines(): - if re_remove.match(i) is None: - to_write.append(i) - f.seek(0) - f.truncate() + try: + for i in f.readlines(): + if re_remove.match(i) is None: + to_write.append(i) + f.seek(0) + f.truncate() - for i in to_write: - f.write(i) - finally: - f.close() + for i in to_write: + f.write(i) + finally: + f.close() logging.info("Successfully cleaned up requirements in " + file_) @@ -427,6 +430,8 @@ def init(args): imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) + # sort imports based on lowercase name of package, similar to `pip freeze`. + imports = sorted(imports, key=lambda x: x['name'].lower()) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) @@ -447,6 +452,9 @@ def init(args): "use --force to overwrite it") return + if args.get('--no-pin'): + imports = [{'name': item["name"], 'version': ''} for item in imports] + if args["--print"]: output_requirements(imports) logging.info("Successfully output requirements") diff --git a/pipreqs/stdlib b/pipreqs/stdlib index 71edcc8..470fd5c 100644 --- a/pipreqs/stdlib +++ b/pipreqs/stdlib @@ -1,6 +1,8 @@ __builtin__ __future__ __main__ +_dummy_thread +_thread _winreg abc aepack @@ -15,6 +17,7 @@ argparse array ast asynchat +asyncio asyncore atexit audioop @@ -28,6 +31,7 @@ binhex bisect bsddb buildtools +builtins bz2 calendar Carbon @@ -101,6 +105,7 @@ code codecs codeop collections +collections.abc ColorPicker colorsys commands @@ -108,12 +113,16 @@ compileall compiler compiler.ast compiler.visitor +concurrent +concurrent.futures ConfigParser +configparser contextlib Cookie cookielib copy copy_reg +copyreg cPickle cProfile crypt @@ -127,6 +136,9 @@ curses.textpad datetime dbhash dbm +dbm.dumb +dbm.gnu +dbm.ndbm decimal DEVICE difflib @@ -188,21 +200,27 @@ dummy_threading EasyDialogs email email.charset +email.contentmanager email.encoders email.errors email.generator email.header +email.headerregistry email.iterators email.message email.mime email.parser +email.policy email.utils encodings encodings.idna +encodings.mbcs encodings.utf_8_sig ensurepip +enum errno exceptions +faulthandler fcntl filecmp fileinput @@ -226,8 +244,8 @@ gensuitemodule getopt getpass gettext -gl GL +gl glob grp gzip @@ -236,9 +254,17 @@ heapq hmac hotshot hotshot.stats +html +html.entities +html.parser htmlentitydefs htmllib HTMLParser +http +http.client +http.cookiejar +http.cookies +http.server httplib ic icopen @@ -248,12 +274,17 @@ imgfile imghdr imp importlib +importlib.abc +importlib.machinery +importlib.util imputil inspect io +ipaddress itertools jpeg json +json.tool keyword lib2to3 linecache @@ -261,6 +292,7 @@ locale logging logging.config logging.handlers +lzma macerrors MacOS macostools @@ -301,11 +333,13 @@ os os.path ossaudiodev parser +pathlib pdb pickle pickletools pipes PixMapWrapper +pkg_resources pkgutil platform plistlib @@ -322,10 +356,12 @@ py_compile pyclbr pydoc Queue +queue quopri random re readline +reprlib resource rexec rfc822 @@ -335,7 +371,9 @@ runpy sched ScrolledText select +selectors sets +setuptools sgmllib sha shelve @@ -350,10 +388,12 @@ smtplib sndhdr socket SocketServer +socketserver spwd sqlite3 ssl stat +statistics statvfs string StringIO @@ -361,8 +401,8 @@ stringprep struct subprocess sunau -sunaudiodev SUNAUDIODEV +sunaudiodev symbol symtable sys @@ -374,6 +414,7 @@ telnetlib tempfile termios test +test.support test.test_support textwrap thread @@ -382,17 +423,30 @@ time timeit Tix Tkinter +tkinter +tkinter.scrolledtext +tkinter.tix +tkinter.ttk token tokenize trace traceback +tracemalloc ttk tty turtle +turtledemo types +typing unicodedata unittest +unittest.mock urllib +urllib.error +urllib.parse +urllib.request +urllib.response +urllib.robotparser urllib2 urlparse user @@ -401,6 +455,7 @@ UserList UserString uu uuid +venv videoreader W warnings @@ -408,6 +463,7 @@ wave weakref webbrowser whichdb +winreg winsound wsgiref wsgiref.handlers @@ -422,654 +478,17 @@ xml.dom.minidom xml.dom.pulldom xml.etree.ElementTree xml.parsers.expat +xml.parsers.expat.errors +xml.parsers.expat.model xml.sax xml.sax.handler xml.sax.saxutils xml.sax.xmlreader +xmlrpc +xmlrpc.client +xmlrpc.server xmlrpclib -zipfile -zipimport -zlib -__future__ -__main__ -_dummy_thread -_thread -abc -aifc -argparse -array -ast -asynchat -asyncio -asyncore -atexit -audioop -base64 -bdb -binascii -binhex -bisect -builtins -bz2 -calendar -cgi -cgitb -chunk -cmath -cmd -code -codecs -codeop -collections -collections.abc -colorsys -compileall -concurrent -concurrent.futures -configparser -contextlib -copy -copyreg -cProfile -crypt -csv -ctypes -curses -curses.ascii -curses.panel -curses.textpad -datetime -dbm -dbm.dumb -dbm.gnu -dbm.ndbm -decimal -difflib -dis -distutils -distutils.archive_util -distutils.bcppcompiler -distutils.ccompiler -distutils.cmd -distutils.command -distutils.command.bdist -distutils.command.bdist_dumb -distutils.command.bdist_msi -distutils.command.bdist_packager -distutils.command.bdist_rpm -distutils.command.bdist_wininst -distutils.command.build -distutils.command.build_clib -distutils.command.build_ext -distutils.command.build_py -distutils.command.build_scripts -distutils.command.check -distutils.command.clean -distutils.command.config -distutils.command.install -distutils.command.install_data -distutils.command.install_headers -distutils.command.install_lib -distutils.command.install_scripts -distutils.command.register -distutils.command.sdist -distutils.core -distutils.cygwinccompiler -distutils.debug -distutils.dep_util -distutils.dir_util -distutils.dist -distutils.errors -distutils.extension -distutils.fancy_getopt -distutils.file_util -distutils.filelist -distutils.log -distutils.msvccompiler -distutils.spawn -distutils.sysconfig -distutils.text_file -distutils.unixccompiler -distutils.util -distutils.version -doctest -dummy_threading -email -email.charset -email.contentmanager -email.encoders -email.errors -email.generator -email.header -email.headerregistry -email.iterators -email.message -email.mime -email.parser -email.policy -email.utils -encodings -encodings.idna -encodings.mbcs -encodings.utf_8_sig -ensurepip -enum -errno -faulthandler -fcntl -filecmp -fileinput -fnmatch -formatter -fpectl -fractions -ftplib -functools -gc -getopt -getpass -gettext -glob -grp -gzip -hashlib -heapq -hmac -html -html.entities -html.parser -http -http.client -http.cookiejar -http.cookies -http.server -imaplib -imghdr -imp -importlib -importlib.abc -importlib.machinery -importlib.util -inspect -io -ipaddress -itertools -json -keyword -lib2to3 -linecache -locale -logging -logging.config -logging.handlers -lzma -macpath -mailbox -mailcap -marshal -math -mimetypes -mmap -modulefinder -msilib -msvcrt -multiprocessing -multiprocessing.connection -multiprocessing.dummy -multiprocessing.managers -multiprocessing.pool -multiprocessing.sharedctypes -netrc -nis -nntplib -numbers -operator -optparse -os -os.path -ossaudiodev -parser -pathlib -pdb -pickle -pickletools -pipes -pkgutil -platform -plistlib -poplib -posix -pprint -profile -pstats -pty -pwd -py_compile -pyclbr -pydoc -queue -quopri -random -re -readline -reprlib -resource -rlcompleter -runpy -sched -select -selectors -shelve -shlex -shutil -signal -site -smtpd -smtplib -sndhdr -socket -socketserver -spwd -sqlite3 -ssl -stat -statistics -string -stringprep -struct -subprocess -sunau -symbol -symtable -sys -sysconfig -syslog -tabnanny -tarfile -telnetlib -tempfile -termios -test -test.support -textwrap -threading -time -timeit -tkinter -tkinter.scrolledtext -tkinter.tix -tkinter.ttk -token -tokenize -trace -traceback -tracemalloc -tty -turtle -turtledemo -types -unicodedata -unittest -unittest.mock -urllib -urllib.error -urllib.parse -urllib.request -urllib.response -urllib.robotparser -uu -uuid -venv -warnings -wave -weakref -webbrowser -winreg -winsound -wsgiref -wsgiref.handlers -wsgiref.headers -wsgiref.simple_server -wsgiref.util -wsgiref.validate -xdrlib -xml -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree.ElementTree -xml.parsers.expat -xml.parsers.expat.errors -xml.parsers.expat.model -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc -xmlrpc.client -xmlrpc.server -zipfile -zipimport -zlib -__future__ -__main__ -_dummy_thread -_thread -abc -aifc -argparse -array -ast -asynchat -asyncio -asyncore -atexit -audioop -base64 -bdb -binascii -binhex -bisect -builtins -bz2 -calendar -cgi -cgitb -chunk -cmath -cmd -code -codecs -codeop -collections -collections.abc -colorsys -compileall -concurrent -concurrent.futures -configparser -contextlib -copy -copyreg -cProfile -crypt -csv -ctypes -curses -curses.ascii -curses.panel -curses.textpad -datetime -dbm -dbm.dumb -dbm.gnu -dbm.ndbm -decimal -difflib -dis -distutils -distutils.archive_util -distutils.bcppcompiler -distutils.ccompiler -distutils.cmd -distutils.command -distutils.command.bdist -distutils.command.bdist_dumb -distutils.command.bdist_msi -distutils.command.bdist_packager -distutils.command.bdist_rpm -distutils.command.bdist_wininst -distutils.command.build -distutils.command.build_clib -distutils.command.build_ext -distutils.command.build_py -distutils.command.build_scripts -distutils.command.check -distutils.command.clean -distutils.command.config -distutils.command.install -distutils.command.install_data -distutils.command.install_headers -distutils.command.install_lib -distutils.command.install_scripts -distutils.command.register -distutils.command.sdist -distutils.core -distutils.cygwinccompiler -distutils.debug -distutils.dep_util -distutils.dir_util -distutils.dist -distutils.errors -distutils.extension -distutils.fancy_getopt -distutils.file_util -distutils.filelist -distutils.log -distutils.msvccompiler -distutils.spawn -distutils.sysconfig -distutils.text_file -distutils.unixccompiler -distutils.util -distutils.version -doctest -dummy_threading -email -email.charset -email.contentmanager -email.encoders -email.errors -email.generator -email.header -email.headerregistry -email.iterators -email.message -email.mime -email.parser -email.policy -email.utils -encodings -encodings.idna -encodings.mbcs -encodings.utf_8_sig -ensurepip -enum -errno -faulthandler -fcntl -filecmp -fileinput -fnmatch -formatter -fpectl -fractions -ftplib -functools -gc -getopt -getpass -gettext -glob -grp -gzip -hashlib -heapq -hmac -html -html.entities -html.parser -http -http.client -http.cookiejar -http.cookies -http.server -imaplib -imghdr -imp -importlib -importlib.abc -importlib.machinery -importlib.util -inspect -io -ipaddress -itertools -json -json.tool -keyword -lib2to3 -linecache -locale -logging -logging.config -logging.handlers -lzma -macpath -mailbox -mailcap -marshal -math -mimetypes -mmap -modulefinder -msilib -msvcrt -multiprocessing -multiprocessing.connection -multiprocessing.dummy -multiprocessing.managers -multiprocessing.pool -multiprocessing.sharedctypes -netrc -nis -nntplib -numbers -operator -optparse -os -os.path -ossaudiodev -parser -pathlib -pdb -pickle -pickletools -pipes -pkgutil -platform -plistlib -poplib -posix -pprint -profile -pstats -pty -pwd -py_compile -pyclbr -pydoc -queue -quopri -random -re -readline -reprlib -resource -rlcompleter -runpy -sched -select -selectors -shelve -shlex -shutil -signal -site -smtpd -smtplib -sndhdr -socket -socketserver -spwd -sqlite3 -ssl -stat -statistics -string -stringprep -struct -subprocess -sunau -symbol -symtable -sys -sysconfig -syslog -tabnanny -tarfile -telnetlib -tempfile -termios -test -test.support -textwrap -threading -time -timeit -tkinter -tkinter.scrolledtext -tkinter.tix -tkinter.ttk -token -tokenize -trace -traceback -tracemalloc -tty -turtle -turtledemo -types -unicodedata -unittest -unittest.mock -urllib -urllib.error -urllib.parse -urllib.request -urllib.response -urllib.robotparser -uu -uuid -venv -warnings -wave -weakref -webbrowser -winreg -winsound -wsgiref -wsgiref.handlers -wsgiref.headers -wsgiref.simple_server -wsgiref.util -wsgiref.validate -xdrlib -xml -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree.ElementTree -xml.parsers.expat -xml.parsers.expat.errors -xml.parsers.expat.model -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc -xmlrpc.client -xmlrpc.server +yp zipapp zipfile zipimport diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index a020fab..68f7249 100755 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -99,6 +99,9 @@ class TestPipreqs(unittest.TestCase): data = f.read().lower() for item in self.modules[:-3]: self.assertTrue(item.lower() in data) + # It should be sorted based on names. + data = data.strip().split('\n') + self.assertEqual(data, sorted(data)) def test_init_local_only(self): """ @@ -181,6 +184,24 @@ class TestPipreqs(unittest.TestCase): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) + def test_omit_version(self): + """ + Test --no-pin parameter + """ + pipreqs.init( + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy': None, + '--pypi-server': None, + '--diff': None, + '--clean': None, + '--no-pin': True + } + ) + with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.read().lower() + for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']: + self.assertFalse(item.lower() in data) def tearDown(self): """