diff --git a/.travis.yml b/.travis.yml index 324b044..7988659 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,13 @@ matrix: env: TOX_ENV=py34 - python: 2.7 env: TOX_ENV=py27 - - python: pypy - env: TOX_ENV=pypy + - python: pypy3 + env: TOX_ENV=pypy3 - python: 3.6 env: TOX_ENV=flake8 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment -install: +install: - pip install coverage coveralls tox # Command to run tests, e.g. python setup.py test diff --git a/AUTHORS.rst b/AUTHORS.rst index eaa038a..2f31bac 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,4 +10,5 @@ Development Lead Contributors ------------ -None yet. Why not be the first? +* Jake Teo +* Jerome Chan diff --git a/HISTORY.rst b/HISTORY.rst index b9d17e7..cb4d47b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,11 @@ History ------- +0.4.11 (2020-03-29) +-------------------- + +* Implement '--mode' (Jake Teo, Jerome Chan) + 0.4.8 (2017-06-30) -------------------- diff --git a/README.rst b/README.rst index 139fa7d..07fee33 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,10 @@ Usage --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --no-pin Omit version of output packages. + --mode Enables dynamic versioning with , or schemes. + | e.g. Flask~=1.1.2 + | e.g. Flask>=1.1.2 + | e.g. Flask Example ------- @@ -70,5 +73,5 @@ Why not pip freeze? ------------------- - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. -- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv) -- and sometimes you just need to create requirements.txt for a new project without installing modules. +- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``). +- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules. diff --git a/pipreqs/mapping b/pipreqs/mapping index 6f5a469..c1729eb 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -1,3 +1,4 @@ +AFQ:pyAFQ AG_fft_tools:agpy ANSI:pexpect Adafruit:Adafruit_Libraries @@ -13,6 +14,7 @@ Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper +functional:pyfunctional GeoBaseMain:GeoBasesDev GeoBases:GeoBasesDev Globals:Zope2 @@ -22,6 +24,7 @@ Kittens:astro_kittens Levenshtein:python_Levenshtein Lifetime:Zope2 MethodObject:ExtensionClass +MySQLdb:MySQL-python OFS:Zope2 OpenGL:PyOpenGL OpenSSL:pyOpenSSL @@ -592,6 +595,7 @@ devtools:tg.devtools dgis:2gis dhtmlparser:pyDHTMLParser digitalocean:python_digitalocean +discord:discord.py distribute_setup:ez_setup distutils2:Distutils2 django:Django @@ -675,6 +679,7 @@ geventwebsocket:gevent_websocket gflags:python_gflags git:GitPython github:PyGithub +github3:github3.py gitpy:git_py globusonline:globusonline_transfer_api_client google:protobuf @@ -701,7 +706,7 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad -krbV:krbv +hydra:hydra-core i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -727,6 +732,7 @@ keyczar:python_keyczar keyedcache:django_keyedcache keystoneclient:python_keystoneclient kickstarter:kickstart +krbv:krbV kss:kss.core kuyruk:Kuyruk langconv:AdvancedLangConv @@ -798,7 +804,6 @@ msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged -MySQL-python:MySQLdb native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -999,6 +1004,7 @@ ruamel:ruamel.base s2repoze:pysaml2 saga:saga_python saml2:pysaml2 +samtranslator:aws-sam-translator sass:libsass sassc:libsass sasstests:libsass diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py old mode 100755 new mode 100644 index 4b817c3..6c936aa --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,7 +31,11 @@ Options: imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --no-pin Omit version of output packages. + --mode Enables dynamic versioning with , + or schemes. + | e.g. Flask~=1.1.2 + | e.g. Flask>=1.1.2 + | e.g. Flask """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -157,25 +161,25 @@ def get_all_imports( return list(packages - data) -def filter_line(l): - return len(l) > 0 and l[0] != "#" +def filter_line(line): + return len(line) > 0 and line[0] != "#" -def generate_requirements_file(path, imports): +def generate_requirements_file(path, imports, symbol): with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, imports=", ".join([x['name'] for x in imports]) )) - fmt = '{name}=={version}' + fmt = '{name}' + symbol + '{version}' out_file.write('\n'.join( fmt.format(**item) if item['version'] else '{name}'.format(**item) for item in imports) + '\n') -def output_requirements(imports): - generate_requirements_file('-', imports) +def output_requirements(imports, symbol): + generate_requirements_file('-', imports, symbol) def get_imports_info( @@ -368,6 +372,11 @@ def diff(file_, imports): def clean(file_, imports): """Remove modules that aren't imported in project from file.""" modules_not_imported = compare_modules(file_, imports) + + if len(modules_not_imported) == 0: + logging.info("Nothing to clean in " + file_) + return + re_remove = re.compile("|".join(modules_not_imported)) to_write = [] @@ -392,6 +401,18 @@ def clean(file_, imports): logging.info("Successfully cleaned up requirements in " + file_) +def dynamic_versioning(scheme, imports): + """Enables dynamic versioning with , or schemes.""" + if scheme == "no-pin": + imports = [{"name": item["name"], "version": ""} for item in imports] + symbol = "" + elif scheme == "gt": + symbol = ">=" + elif scheme == "compat": + symbol = "~=" + return imports, symbol + + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -430,6 +451,8 @@ def init(args): imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) + # sort imports based on lowercase name of package, similar to `pip freeze`. + imports = sorted(imports, key=lambda x: x['name'].lower()) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) @@ -446,18 +469,25 @@ def init(args): and not args["--savepath"] and not args["--force"] and os.path.exists(path)): - logging.warning("Requirements.txt already exists, " + logging.warning("requirements.txt already exists, " "use --force to overwrite it") return - if args.get('--no-pin'): - imports = [{'name': item["name"], 'version': ''} for item in imports] + if args["--mode"]: + scheme = args.get("--mode") + if scheme in ["compat", "gt", "no-pin"]: + imports, symbol = dynamic_versioning(scheme, imports) + else: + raise ValueError("Invalid argument for mode flag, " + "use 'compat', 'gt' or 'no-pin' instead") + else: + symbol = "==" if args["--print"]: - output_requirements(imports) + output_requirements(imports, symbol) logging.info("Successfully output requirements") else: - generate_requirements_file(path, imports) + generate_requirements_file(path, imports, symbol) logging.info("Successfully saved requirements file in " + path) diff --git a/pipreqs/stdlib b/pipreqs/stdlib index 470fd5c..c2c1fdd 100644 --- a/pipreqs/stdlib +++ b/pipreqs/stdlib @@ -133,6 +133,7 @@ curses curses.ascii curses.panel curses.textpad +dataclasses datetime dbhash dbm @@ -370,6 +371,7 @@ robotparser runpy sched ScrolledText +secrets select selectors sets diff --git a/tests/_data/test.py b/tests/_data/test.py index cfd039c..fdb6ec3 100644 --- a/tests/_data/test.py +++ b/tests/_data/test.py @@ -31,6 +31,10 @@ from pyflakes.test.test_imports import Test as TestImports # Nose from nose.importer import Importer, add_path, remove_path # loader.py +# see issue #88 +import analytics +import flask_seasurf + import atexit from __future__ import print_function from docopt import docopt diff --git a/tests/_data_clean/test.py b/tests/_data_clean/test.py new file mode 100644 index 0000000..8cffb51 --- /dev/null +++ b/tests/_data_clean/test.py @@ -0,0 +1,65 @@ +"""unused import""" +# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring +import xml.etree # [unused-import] +import xml.sax # [unused-import] +import os.path as test # [unused-import] +from sys import argv as test2 # [unused-import] +from sys import flags # [unused-import] +# +1:[unused-import,unused-import] +from collections import deque, OrderedDict, Counter +# All imports above should be ignored +import requests # [unused-import] + +# setuptools +import zipimport # command/easy_install.py + +# twisted +from importlib import invalidate_caches # python/test/test_deprecate.py + +# astroid +import zipimport # manager.py +# IPython +from importlib.machinery import all_suffixes # core/completerlib.py +import importlib # html/notebookapp.py + +from IPython.utils.importstring import import_item # Many files + +# pyflakes +# test/test_doctests.py +from pyflakes.test.test_imports import Test as TestImports + +# Nose +from nose.importer import Importer, add_path, remove_path # loader.py + +# see issue #88 +import analytics +import flask_seasurf + +import atexit +from __future__ import print_function +from docopt import docopt +import curses, logging, sqlite3 +import logging +import os +import sqlite3 +import time +import sys +import signal +import bs4 +import nonexistendmodule +import boto as b, peewee as p +# import django +import flask.ext.somext # # # +# from sqlalchemy import model +try: + import ujson as json +except ImportError: + import json + +import models + + +def main(): + pass + +import after_method_is_valid_even_if_not_pep8 diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py old mode 100755 new mode 100644 index dcd75c5..f82d3db --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -18,22 +18,40 @@ from pipreqs import pipreqs class TestPipreqs(unittest.TestCase): def setUp(self): - self.modules = ['flask', 'requests', 'sqlalchemy', - 'docopt', 'boto', 'ipython', 'pyflakes', 'nose', - 'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ] + self.modules = [ + 'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython', + 'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee', + 'ujson', 'nonexistendmodule', 'bs4', + 'after_method_is_valid_even_if_not_pep8' + ] self.modules2 = ['beautifulsoup4'] self.local = ["docopt", "requests", "nose", 'pyflakes'] self.project = os.path.join(os.path.dirname(__file__), "_data") - self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data") - self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore") - self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps") + self.project_clean = os.path.join( + os.path.dirname(__file__), + "_data_clean" + ) + self.project_invalid = os.path.join( + os.path.dirname(__file__), + "_invalid_data" + ) + self.project_with_ignore_directory = os.path.join( + os.path.dirname(__file__), + "_data_ignore" + ) + self.project_with_duplicated_deps = os.path.join( + os.path.dirname(__file__), + "_data_duplicated_deps" + ) self.requirements_path = os.path.join(self.project, "requirements.txt") self.alt_requirement_path = os.path.join( - self.project, "requirements2.txt") + self.project, + "requirements2.txt" + ) def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) - self.assertEqual(len(imports), 13) + self.assertEqual(len(imports), 15) for item in imports: self.assertTrue( item.lower() in self.modules, "Import is missing: " + item) @@ -54,7 +72,8 @@ class TestPipreqs(unittest.TestCase): """ Test that invalid python files cannot be imported. """ - self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid) + self.assertRaises( + SyntaxError, pipreqs.get_all_imports, self.project_invalid) def test_get_imports_info(self): """ @@ -62,8 +81,9 @@ class TestPipreqs(unittest.TestCase): """ imports = pipreqs.get_all_imports(self.project) with_info = pipreqs.get_imports_info(imports) - # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8" - self.assertEqual(len(with_info), 11) + # Should contain 10 items without the "nonexistendmodule" and + # "after_method_is_valid_even_if_not_pep8" + self.assertEqual(len(with_info), 13) for item in with_info: self.assertTrue( item['name'].lower() in self.modules, @@ -77,10 +97,12 @@ class TestPipreqs(unittest.TestCase): def test_get_use_local_only(self): """ - Test without checking PyPI, check to see if names of local imports matches what we expect + Test without checking PyPI, check to see if names of local + imports matches what we expect - Note even though pyflakes isn't in requirements.txt, - It's added to locals since it is a development dependency for testing + It's added to locals since it is a development dependency + for testing """ # should find only docopt and requests imports_with_info = pipreqs.get_import_local(self.modules) @@ -89,24 +111,28 @@ class TestPipreqs(unittest.TestCase): def test_init(self): """ - Test that all modules we will test upon, are in requirements file + Test that all modules we will test upon are in requirements file """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None}) + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: self.assertTrue(item.lower() in data) + # It should be sorted based on names. + data = data.strip().split('\n') + self.assertEqual(data, sorted(data)) def test_init_local_only(self): """ - Test that items listed in requirements.text are the same as locals expected + Test that items listed in requirements.text are the same + as locals expected """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None}) + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -116,11 +142,12 @@ class TestPipreqs(unittest.TestCase): def test_init_savepath(self): """ - Test that we can save requiremnts.tt correctly to a different path + Test that we can save requirements.txt correctly + to a different path """ - pipreqs.init({'': self.project, '--savepath': - self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None}) + pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, + '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -131,13 +158,14 @@ class TestPipreqs(unittest.TestCase): def test_init_overwrite(self): """ - Test that if requiremnts.txt exists, it will not automatically be overwritten + Test that if requiremnts.txt exists, it will not be + automatically overwritten """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, - '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None}) + pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, + '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -145,35 +173,43 @@ class TestPipreqs(unittest.TestCase): def test_get_import_name_without_alias(self): """ - Test that function get_name_without_alias() will work on a string. - - Note: This isn't truly needed when pipreqs is walking the AST to find imports + Test that function get_name_without_alias() + will work on a string. + - Note: This isn't truly needed when pipreqs is walking + the AST to find imports """ import_name_with_alias = "requests as R" expected_import_name_without_alias = "requests" import_name_without_aliases = pipreqs.get_name_without_alias( import_name_with_alias) self.assertEqual( - import_name_without_aliases, expected_import_name_without_alias) + import_name_without_aliases, + expected_import_name_without_alias + ) def test_custom_pypi_server(self): """ Test that trying to get a custom pypi sever fails correctly """ - self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'}) + self.assertRaises( + requests.exceptions.MissingSchema, pipreqs.init, + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': 'nonexistent'} + ) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy':None, - '--pypi-server':None, + {'': self.project_with_ignore_directory, '--savepath': None, + '--print': False, '--use-local': None, '--force': True, + '--proxy':None, '--pypi-server':None, '--ignore':'.ignored_dir,.ignore_second', '--diff': None, - '--clean': None + '--clean': None, + '--mode': None } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -181,9 +217,27 @@ class TestPipreqs(unittest.TestCase): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) - def test_omit_version(self): + def test_dynamic_version_no_pin_scheme(self): """ - Test --no-pin parameter + Test --mode=no-pin + """ + pipreqs.init( + {'': self.project_with_ignore_directory, '--savepath': None, + '--print': False, '--use-local': None, '--force': True, + '--proxy': None, '--pypi-server': None, + '--diff': None, + '--clean': None, + '--mode': 'no-pin' + } + ) + with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.read().lower() + for item in ['beautifulsoup4', 'boto']: + self.assertTrue(item.lower() in data) + + def test_dynamic_version_gt_scheme(self): + """ + Test --mode=gt """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -192,13 +246,81 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--no-pin': True + '--mode': 'gt' } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.readlines() + for item in data: + symbol = '>=' + message = 'symbol is not in item' + self.assertIn(symbol, item, message) + + def test_dynamic_version_compat_scheme(self): + """ + Test --mode=compat + """ + pipreqs.init( + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy': None, + '--pypi-server': None, + '--diff': None, + '--clean': None, + '--mode': 'compat' + } + ) + with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.readlines() + for item in data: + symbol = '~=' + message = 'symbol is not in item' + self.assertIn(symbol, item, message) + + def test_clean(self): + """ + Test --clean parameter + """ + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None, + '--mode': None} + ) + assert os.path.exists(self.requirements_path) == 1 + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': None, '--proxy': None, + '--pypi-server': None, '--diff': None, + '--clean': self.requirements_path, '--mode': 'non-pin'} + ) + with open(self.requirements_path, "r") as f: data = f.read().lower() - for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']: - self.assertFalse(item.lower() in data) + for item in self.modules[:-3]: + self.assertTrue(item.lower() in data) + + def test_clean_with_imports_to_clean(self): + """ + Test --clean parameter when there are imports to clean + """ + cleaned_module = 'sqlalchemy' + pipreqs.init( + {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, + '--pypi-server': None, '--diff': None, '--clean': None, + '--mode': None} + ) + assert os.path.exists(self.requirements_path) == 1 + modules_clean = [m for m in self.modules if m != cleaned_module] + pipreqs.init( + {'': self.project_clean, '--savepath': None, + '--print': False, '--use-local': None, '--force': None, + '--proxy': None, '--pypi-server': None, '--diff': None, + '--clean': self.requirements_path, '--mode': 'non-pin'} + ) + with open(self.requirements_path, "r") as f: + data = f.read().lower() + self.assertTrue(cleaned_module not in data) def tearDown(self): """ diff --git a/tox.ini b/tox.ini index 28a1dfa..554b3c1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy, flake8 +envlist = py27, py34, py35, py36, pypy3, flake8 [testenv] setenv =