From 90102acdbb23c09574d27df8bd1f568d34e0cfd3 Mon Sep 17 00:00:00 2001 From: alan-barzilay Date: Tue, 4 May 2021 20:39:47 -0300 Subject: [PATCH] Revert all commits since last release By reverting all commits done since release v0.4.10 we will have the master branch synchronized with the latest release available in pipy. All commits done since the latest release will be moved to another branch called `next` where we will centralize development. Once we are ready for a new release of pipreqs, the `next` branch will be merged back on to master and a new release will be made. This change will make development more organized and will avoid new issues from users complaining about features only present in master not working on their installation of pipreqs. I would also like to thank @pedroteosousa for his help on reverting and squashing all commits --- .travis.yml | 6 +- AUTHORS.rst | 3 +- HISTORY.rst | 5 - README.rst | 9 +- pipreqs/mapping | 10 +- pipreqs/pipreqs.py | 54 +++------- pipreqs/stdlib | 2 - tests/_data/test.py | 4 - tests/_data_clean/test.py | 65 ------------ tests/test_pipreqs.py | 202 ++++++++------------------------------ tox.ini | 2 +- 11 files changed, 62 insertions(+), 300 deletions(-) mode change 100644 => 100755 pipreqs/pipreqs.py delete mode 100644 tests/_data_clean/test.py mode change 100644 => 100755 tests/test_pipreqs.py diff --git a/.travis.yml b/.travis.yml index 7988659..324b044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,13 +12,13 @@ matrix: env: TOX_ENV=py34 - python: 2.7 env: TOX_ENV=py27 - - python: pypy3 - env: TOX_ENV=pypy3 + - python: pypy + env: TOX_ENV=pypy - python: 3.6 env: TOX_ENV=flake8 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment -install: +install: - pip install coverage coveralls tox # Command to run tests, e.g. python setup.py test diff --git a/AUTHORS.rst b/AUTHORS.rst index 2f31bac..eaa038a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,5 +10,4 @@ Development Lead Contributors ------------ -* Jake Teo -* Jerome Chan +None yet. Why not be the first? diff --git a/HISTORY.rst b/HISTORY.rst index cb4d47b..b9d17e7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,11 +3,6 @@ History ------- -0.4.11 (2020-03-29) --------------------- - -* Implement '--mode' (Jake Teo, Jerome Chan) - 0.4.8 (2017-06-30) -------------------- diff --git a/README.rst b/README.rst index 07fee33..139fa7d 100644 --- a/README.rst +++ b/README.rst @@ -49,10 +49,7 @@ Usage --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --mode Enables dynamic versioning with , or schemes. - | e.g. Flask~=1.1.2 - | e.g. Flask>=1.1.2 - | e.g. Flask + --no-pin Omit version of output packages. Example ------- @@ -73,5 +70,5 @@ Why not pip freeze? ------------------- - ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment. -- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``). -- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules. +- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv) +- and sometimes you just need to create requirements.txt for a new project without installing modules. diff --git a/pipreqs/mapping b/pipreqs/mapping index c1729eb..6f5a469 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -1,4 +1,3 @@ -AFQ:pyAFQ AG_fft_tools:agpy ANSI:pexpect Adafruit:Adafruit_Libraries @@ -14,7 +13,6 @@ Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper -functional:pyfunctional GeoBaseMain:GeoBasesDev GeoBases:GeoBasesDev Globals:Zope2 @@ -24,7 +22,6 @@ Kittens:astro_kittens Levenshtein:python_Levenshtein Lifetime:Zope2 MethodObject:ExtensionClass -MySQLdb:MySQL-python OFS:Zope2 OpenGL:PyOpenGL OpenSSL:pyOpenSSL @@ -595,7 +592,6 @@ devtools:tg.devtools dgis:2gis dhtmlparser:pyDHTMLParser digitalocean:python_digitalocean -discord:discord.py distribute_setup:ez_setup distutils2:Distutils2 django:Django @@ -679,7 +675,6 @@ geventwebsocket:gevent_websocket gflags:python_gflags git:GitPython github:PyGithub -github3:github3.py gitpy:git_py globusonline:globusonline_transfer_api_client google:protobuf @@ -706,7 +701,7 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad -hydra:hydra-core +krbV:krbv i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -732,7 +727,6 @@ keyczar:python_keyczar keyedcache:django_keyedcache keystoneclient:python_keystoneclient kickstarter:kickstart -krbv:krbV kss:kss.core kuyruk:Kuyruk langconv:AdvancedLangConv @@ -804,6 +798,7 @@ msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged +MySQL-python:MySQLdb native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -1004,7 +999,6 @@ ruamel:ruamel.base s2repoze:pysaml2 saga:saga_python saml2:pysaml2 -samtranslator:aws-sam-translator sass:libsass sassc:libsass sasstests:libsass diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py old mode 100644 new mode 100755 index 6c936aa..4b817c3 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,11 +31,7 @@ Options: imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --mode Enables dynamic versioning with , - or schemes. - | e.g. Flask~=1.1.2 - | e.g. Flask>=1.1.2 - | e.g. Flask + --no-pin Omit version of output packages. """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -161,25 +157,25 @@ def get_all_imports( return list(packages - data) -def filter_line(line): - return len(line) > 0 and line[0] != "#" +def filter_line(l): + return len(l) > 0 and l[0] != "#" -def generate_requirements_file(path, imports, symbol): +def generate_requirements_file(path, imports): with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, imports=", ".join([x['name'] for x in imports]) )) - fmt = '{name}' + symbol + '{version}' + fmt = '{name}=={version}' out_file.write('\n'.join( fmt.format(**item) if item['version'] else '{name}'.format(**item) for item in imports) + '\n') -def output_requirements(imports, symbol): - generate_requirements_file('-', imports, symbol) +def output_requirements(imports): + generate_requirements_file('-', imports) def get_imports_info( @@ -372,11 +368,6 @@ def diff(file_, imports): def clean(file_, imports): """Remove modules that aren't imported in project from file.""" modules_not_imported = compare_modules(file_, imports) - - if len(modules_not_imported) == 0: - logging.info("Nothing to clean in " + file_) - return - re_remove = re.compile("|".join(modules_not_imported)) to_write = [] @@ -401,18 +392,6 @@ def clean(file_, imports): logging.info("Successfully cleaned up requirements in " + file_) -def dynamic_versioning(scheme, imports): - """Enables dynamic versioning with , or schemes.""" - if scheme == "no-pin": - imports = [{"name": item["name"], "version": ""} for item in imports] - symbol = "" - elif scheme == "gt": - symbol = ">=" - elif scheme == "compat": - symbol = "~=" - return imports, symbol - - def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -451,8 +430,6 @@ def init(args): imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) - # sort imports based on lowercase name of package, similar to `pip freeze`. - imports = sorted(imports, key=lambda x: x['name'].lower()) path = (args["--savepath"] if args["--savepath"] else os.path.join(input_path, "requirements.txt")) @@ -469,25 +446,18 @@ def init(args): and not args["--savepath"] and not args["--force"] and os.path.exists(path)): - logging.warning("requirements.txt already exists, " + logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return - if args["--mode"]: - scheme = args.get("--mode") - if scheme in ["compat", "gt", "no-pin"]: - imports, symbol = dynamic_versioning(scheme, imports) - else: - raise ValueError("Invalid argument for mode flag, " - "use 'compat', 'gt' or 'no-pin' instead") - else: - symbol = "==" + if args.get('--no-pin'): + imports = [{'name': item["name"], 'version': ''} for item in imports] if args["--print"]: - output_requirements(imports, symbol) + output_requirements(imports) logging.info("Successfully output requirements") else: - generate_requirements_file(path, imports, symbol) + generate_requirements_file(path, imports) logging.info("Successfully saved requirements file in " + path) diff --git a/pipreqs/stdlib b/pipreqs/stdlib index c2c1fdd..470fd5c 100644 --- a/pipreqs/stdlib +++ b/pipreqs/stdlib @@ -133,7 +133,6 @@ curses curses.ascii curses.panel curses.textpad -dataclasses datetime dbhash dbm @@ -371,7 +370,6 @@ robotparser runpy sched ScrolledText -secrets select selectors sets diff --git a/tests/_data/test.py b/tests/_data/test.py index fdb6ec3..cfd039c 100644 --- a/tests/_data/test.py +++ b/tests/_data/test.py @@ -31,10 +31,6 @@ from pyflakes.test.test_imports import Test as TestImports # Nose from nose.importer import Importer, add_path, remove_path # loader.py -# see issue #88 -import analytics -import flask_seasurf - import atexit from __future__ import print_function from docopt import docopt diff --git a/tests/_data_clean/test.py b/tests/_data_clean/test.py deleted file mode 100644 index 8cffb51..0000000 --- a/tests/_data_clean/test.py +++ /dev/null @@ -1,65 +0,0 @@ -"""unused import""" -# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring -import xml.etree # [unused-import] -import xml.sax # [unused-import] -import os.path as test # [unused-import] -from sys import argv as test2 # [unused-import] -from sys import flags # [unused-import] -# +1:[unused-import,unused-import] -from collections import deque, OrderedDict, Counter -# All imports above should be ignored -import requests # [unused-import] - -# setuptools -import zipimport # command/easy_install.py - -# twisted -from importlib import invalidate_caches # python/test/test_deprecate.py - -# astroid -import zipimport # manager.py -# IPython -from importlib.machinery import all_suffixes # core/completerlib.py -import importlib # html/notebookapp.py - -from IPython.utils.importstring import import_item # Many files - -# pyflakes -# test/test_doctests.py -from pyflakes.test.test_imports import Test as TestImports - -# Nose -from nose.importer import Importer, add_path, remove_path # loader.py - -# see issue #88 -import analytics -import flask_seasurf - -import atexit -from __future__ import print_function -from docopt import docopt -import curses, logging, sqlite3 -import logging -import os -import sqlite3 -import time -import sys -import signal -import bs4 -import nonexistendmodule -import boto as b, peewee as p -# import django -import flask.ext.somext # # # -# from sqlalchemy import model -try: - import ujson as json -except ImportError: - import json - -import models - - -def main(): - pass - -import after_method_is_valid_even_if_not_pep8 diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py old mode 100644 new mode 100755 index f82d3db..dcd75c5 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -18,40 +18,22 @@ from pipreqs import pipreqs class TestPipreqs(unittest.TestCase): def setUp(self): - self.modules = [ - 'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython', - 'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee', - 'ujson', 'nonexistendmodule', 'bs4', - 'after_method_is_valid_even_if_not_pep8' - ] + self.modules = ['flask', 'requests', 'sqlalchemy', + 'docopt', 'boto', 'ipython', 'pyflakes', 'nose', + 'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ] self.modules2 = ['beautifulsoup4'] self.local = ["docopt", "requests", "nose", 'pyflakes'] self.project = os.path.join(os.path.dirname(__file__), "_data") - self.project_clean = os.path.join( - os.path.dirname(__file__), - "_data_clean" - ) - self.project_invalid = os.path.join( - os.path.dirname(__file__), - "_invalid_data" - ) - self.project_with_ignore_directory = os.path.join( - os.path.dirname(__file__), - "_data_ignore" - ) - self.project_with_duplicated_deps = os.path.join( - os.path.dirname(__file__), - "_data_duplicated_deps" - ) + self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data") + self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore") + self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps") self.requirements_path = os.path.join(self.project, "requirements.txt") self.alt_requirement_path = os.path.join( - self.project, - "requirements2.txt" - ) + self.project, "requirements2.txt") def test_get_all_imports(self): imports = pipreqs.get_all_imports(self.project) - self.assertEqual(len(imports), 15) + self.assertEqual(len(imports), 13) for item in imports: self.assertTrue( item.lower() in self.modules, "Import is missing: " + item) @@ -72,8 +54,7 @@ class TestPipreqs(unittest.TestCase): """ Test that invalid python files cannot be imported. """ - self.assertRaises( - SyntaxError, pipreqs.get_all_imports, self.project_invalid) + self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid) def test_get_imports_info(self): """ @@ -81,9 +62,8 @@ class TestPipreqs(unittest.TestCase): """ imports = pipreqs.get_all_imports(self.project) with_info = pipreqs.get_imports_info(imports) - # Should contain 10 items without the "nonexistendmodule" and - # "after_method_is_valid_even_if_not_pep8" - self.assertEqual(len(with_info), 13) + # Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8" + self.assertEqual(len(with_info), 11) for item in with_info: self.assertTrue( item['name'].lower() in self.modules, @@ -97,12 +77,10 @@ class TestPipreqs(unittest.TestCase): def test_get_use_local_only(self): """ - Test without checking PyPI, check to see if names of local - imports matches what we expect + Test without checking PyPI, check to see if names of local imports matches what we expect - Note even though pyflakes isn't in requirements.txt, - It's added to locals since it is a development dependency - for testing + It's added to locals since it is a development dependency for testing """ # should find only docopt and requests imports_with_info = pipreqs.get_import_local(self.modules) @@ -111,28 +89,24 @@ class TestPipreqs(unittest.TestCase): def test_init(self): """ - Test that all modules we will test upon are in requirements file + Test that all modules we will test upon, are in requirements file """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() for item in self.modules[:-3]: self.assertTrue(item.lower() in data) - # It should be sorted based on names. - data = data.strip().split('\n') - self.assertEqual(data, sorted(data)) def test_init_local_only(self): """ - Test that items listed in requirements.text are the same - as locals expected + Test that items listed in requirements.text are the same as locals expected """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--mode': None}) + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -142,12 +116,11 @@ class TestPipreqs(unittest.TestCase): def test_init_savepath(self): """ - Test that we can save requirements.txt correctly - to a different path + Test that we can save requiremnts.tt correctly to a different path """ - pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, - '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init({'': self.project, '--savepath': + self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -158,14 +131,13 @@ class TestPipreqs(unittest.TestCase): def test_init_overwrite(self): """ - Test that if requiremnts.txt exists, it will not be - automatically overwritten + Test that if requiremnts.txt exists, it will not automatically be overwritten """ with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") - pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, - '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - '--diff': None, '--clean': None, '--mode': None}) + pipreqs.init({'': self.project, '--savepath': None, + '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -173,43 +145,35 @@ class TestPipreqs(unittest.TestCase): def test_get_import_name_without_alias(self): """ - Test that function get_name_without_alias() - will work on a string. - - Note: This isn't truly needed when pipreqs is walking - the AST to find imports + Test that function get_name_without_alias() will work on a string. + - Note: This isn't truly needed when pipreqs is walking the AST to find imports """ import_name_with_alias = "requests as R" expected_import_name_without_alias = "requests" import_name_without_aliases = pipreqs.get_name_without_alias( import_name_with_alias) self.assertEqual( - import_name_without_aliases, - expected_import_name_without_alias - ) + import_name_without_aliases, expected_import_name_without_alias) def test_custom_pypi_server(self): """ Test that trying to get a custom pypi sever fails correctly """ - self.assertRaises( - requests.exceptions.MissingSchema, pipreqs.init, - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': 'nonexistent'} - ) + self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'': self.project, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'}) def test_ignored_directory(self): """ Test --ignore parameter """ pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy':None, '--pypi-server':None, + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy':None, + '--pypi-server':None, '--ignore':'.ignored_dir,.ignore_second', '--diff': None, - '--clean': None, - '--mode': None + '--clean': None } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -217,27 +181,9 @@ class TestPipreqs(unittest.TestCase): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) - def test_dynamic_version_no_pin_scheme(self): + def test_omit_version(self): """ - Test --mode=no-pin - """ - pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, - '--print': False, '--use-local': None, '--force': True, - '--proxy': None, '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'no-pin' - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.read().lower() - for item in ['beautifulsoup4', 'boto']: - self.assertTrue(item.lower() in data) - - def test_dynamic_version_gt_scheme(self): - """ - Test --mode=gt + Test --no-pin parameter """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -246,81 +192,13 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': 'gt' + '--no-pin': True } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.readlines() - for item in data: - symbol = '>=' - message = 'symbol is not in item' - self.assertIn(symbol, item, message) - - def test_dynamic_version_compat_scheme(self): - """ - Test --mode=compat - """ - pipreqs.init( - {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, - '--proxy': None, - '--pypi-server': None, - '--diff': None, - '--clean': None, - '--mode': 'compat' - } - ) - with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: - data = f.readlines() - for item in data: - symbol = '~=' - message = 'symbol is not in item' - self.assertIn(symbol, item, message) - - def test_clean(self): - """ - Test --clean parameter - """ - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) - assert os.path.exists(self.requirements_path) == 1 - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': None, '--proxy': None, - '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) - with open(self.requirements_path, "r") as f: data = f.read().lower() - for item in self.modules[:-3]: - self.assertTrue(item.lower() in data) - - def test_clean_with_imports_to_clean(self): - """ - Test --clean parameter when there are imports to clean - """ - cleaned_module = 'sqlalchemy' - pipreqs.init( - {'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy': None, - '--pypi-server': None, '--diff': None, '--clean': None, - '--mode': None} - ) - assert os.path.exists(self.requirements_path) == 1 - modules_clean = [m for m in self.modules if m != cleaned_module] - pipreqs.init( - {'': self.project_clean, '--savepath': None, - '--print': False, '--use-local': None, '--force': None, - '--proxy': None, '--pypi-server': None, '--diff': None, - '--clean': self.requirements_path, '--mode': 'non-pin'} - ) - with open(self.requirements_path, "r") as f: - data = f.read().lower() - self.assertTrue(cleaned_module not in data) + for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']: + self.assertFalse(item.lower() in data) def tearDown(self): """ diff --git a/tox.ini b/tox.ini index 554b3c1..28a1dfa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy3, flake8 +envlist = py27, py34, py35, py36, pypy, flake8 [testenv] setenv =