From d8c94d1690157c1c4c74f0506095e4824bff468b Mon Sep 17 00:00:00 2001 From: Zhiming Wang Date: Fri, 8 Sep 2017 22:47:44 -0400 Subject: [PATCH 1/9] fix(pipreqs/mapping): correct arrow mapping https://pypi.org/project/arrow/ is https://github.com/crsmithdev/arrow/, the real arrow. https://pypi.org/project/arrow-fatisar/ is https://github.com/fatisar/arrow, a completely random, outdated fork. --- pipreqs/mapping | 1 - 1 file changed, 1 deletion(-) diff --git a/pipreqs/mapping b/pipreqs/mapping index 46cca29..00362fe 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -263,7 +263,6 @@ armstrong:armstrong.hatband armstrong:armstrong.templates.standard armstrong:armstrong.utils.backends armstrong:armstrong.utils.celery -arrow:arrow_fatisar arstecnica:arstecnica.raccoon.autobahn arstecnica:arstecnica.sqlalchemy.async article-downloader:article_downloader From c80d06593d9b931f2c74c84cc6828cfb3a2626be Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Sun, 22 Oct 2017 23:45:16 -0400 Subject: [PATCH 2/9] Support optional argument This change makes the argument optional, defaulting to the current working directory if omitted. --- Ideally, this would have been accomplished via docopt, but optional positional arguments with defaults are not supported at the moment [1, 2]. [1] https://github.com/docopt/docopt/issues/214 [2] https://github.com/docopt/docopt/issues/329 --- pipreqs/pipreqs.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 2fa71ac..ce08fd5 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -3,7 +3,12 @@ """pipreqs - Generate pip requirements.txt file based on imports Usage: - pipreqs [options] + pipreqs [options] [] + +Arguments: + The path to the directory containing the application + files for which a requirements file should be + generated (defaults to the current working directory). Options: --use-local Use ONLY local package info instead of querying PyPI @@ -335,11 +340,14 @@ def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') follow_links = not args.get('--no-follow-links') + input_path = args[''] + if input_path is None: + input_path = os.path.abspath(os.curdir) if extra_ignore_dirs: extra_ignore_dirs = extra_ignore_dirs.split(',') - candidates = get_all_imports(args[''], + candidates = get_all_imports(input_path, encoding=encoding, extra_ignore_dirs=extra_ignore_dirs, follow_links=follow_links) @@ -368,7 +376,7 @@ def init(args): pypi_server=pypi_server) path = (args["--savepath"] if args["--savepath"] else - os.path.join(args[''], "requirements.txt")) + os.path.join(input_path, "requirements.txt")) if args["--diff"]: diff(args["--diff"], imports) From 0a9845d87d5960c53d23189af349d7bf4fc341b8 Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Tue, 24 Oct 2017 01:22:54 -0400 Subject: [PATCH 3/9] Run Travis-CI tests inside of tox By using tox instead of the default Travis-CI Python environments, we ensure that we have a single entrypoint to testing both locally and in CI. This reduces redundant code and makes it clear when test environments don't match up on different platforms. [tox-travis](https://tox-travis.readthedocs.io/en/stable/) is introduced here to automatically run tox jobs under the proper Travis-CI environments. Additionally, the coveralls step is moved to a [build stage](https://docs.travis-ci.com/user/build-stages) to run once after all other Travis-CI tests complete. --- .travis.yml | 20 ++++++++++++-------- tox.ini | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a7194b..98ac6f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,17 @@ python: - "2.7" - "pypy" -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +# Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment install: - - "pip install -r requirements.txt" - - "pip install coverage" - - "pip install coveralls" + - pip install coverage coveralls tox-travis -# command to run tests, e.g. python setup.py test -script: coverage run --source=pipreqs setup.py test -after_success: - coveralls +# Command to run tests, e.g. python setup.py test +script: tox + +# Use a build stage instead of after_success to get a single coveralls report +jobs: + include: + - stage: Coveralls + script: + - coverage run --source=pipreqs setup.py test + - coveralls diff --git a/tox.ini b/tox.ini index 69eed48..8b6f824 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36 +envlist = py27, py34, py35, py36, pypy [testenv] setenv = From d1a7eda5e8f061b336239a93cc94ebbccd2db13c Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Tue, 24 Oct 2017 00:17:42 -0400 Subject: [PATCH 4/9] Enable flake8 linting in tox.ini and .travis.yml Currently, flake8 is accessible via `make lint`, but it does not run along side the rest of the test suite. This change adds flake8 checks to the tox.ini file to enable linting as a routine part of running tests. Additionally, drop the changes made in #100. --- .travis.yml | 35 ++++++++++++++++++++--------------- tox.ini | 9 ++++++++- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98ac6f0..324b044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,24 +2,29 @@ language: python -python: - - "3.6" - - "3.5" - - "3.4" - - "2.7" - - "pypy" +matrix: + include: + - python: 3.6 + env: TOX_ENV=py36 + - python: 3.5 + env: TOX_ENV=py35 + - python: 3.4 + env: TOX_ENV=py34 + - python: 2.7 + env: TOX_ENV=py27 + - python: pypy + env: TOX_ENV=pypy + - python: 3.6 + env: TOX_ENV=flake8 # Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment install: - - pip install coverage coveralls tox-travis + - pip install coverage coveralls tox # Command to run tests, e.g. python setup.py test -script: tox +script: tox -e $TOX_ENV -# Use a build stage instead of after_success to get a single coveralls report -jobs: - include: - - stage: Coveralls - script: - - coverage run --source=pipreqs setup.py test - - coveralls +# Use after_success to get a single coveralls report +after_success: + - coverage run --source=pipreqs setup.py test + - coveralls diff --git a/tox.ini b/tox.ini index 8b6f824..28a1dfa 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, pypy +envlist = py27, py34, py35, py36, pypy, flake8 [testenv] setenv = @@ -7,3 +7,10 @@ setenv = commands = python setup.py test deps = -r{toxinidir}/requirements.txt + +[testenv:flake8] +basepython = python3.6 +commands = flake8 pipreqs +deps = + -r{toxinidir}/requirements.txt + flake8 From e88b5ab19cb5044acb5a05218865f4d5c1b8999f Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Tue, 24 Oct 2017 01:09:35 -0400 Subject: [PATCH 5/9] Fix flake8 errors --- pipreqs/pipreqs.py | 88 +++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index ce08fd5..d21b071 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -8,24 +8,29 @@ Usage: Arguments: The path to the directory containing the application files for which a requirements file should be - generated (defaults to the current working directory). + generated (defaults to the current working + directory). Options: - --use-local Use ONLY local package info instead of querying PyPI - --pypi-server Use custom PyPi server - --proxy Use Proxy, parameter will be passed to requests library. You can also just set the - environments parameter in your terminal: + --use-local Use ONLY local package info instead of querying PyPI. + --pypi-server Use custom PyPi server. + --proxy Use Proxy, parameter will be passed to requests + library. You can also just set the environments + parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" - --debug Print debug information - --ignore ... Ignore extra directories, each separated by a comma + --debug Print debug information. + --ignore ... Ignore extra directories, each separated by a comma. --no-follow-links Do not follow symbolic links in the project --encoding Use encoding parameter for file open --savepath Save the list of requirements in the given file - --print Output the list of requirements in the standard output + --print Output the list of requirements in the standard + output. --force Overwrite existing requirements.txt - --diff Compare modules in requirements.txt to project imports. - --clean Clean up requirements.txt by removing modules that are not imported in project. + --diff Compare modules in requirements.txt to project + imports. + --clean Clean up requirements.txt by removing modules + that are not imported in project. """ from __future__ import print_function, absolute_import import os @@ -56,7 +61,8 @@ else: py2_exclude = ["concurrent", "concurrent.futures"] -def get_all_imports(path, encoding=None, extra_ignore_dirs=None, follow_links=True): +def get_all_imports( + path, encoding=None, extra_ignore_dirs=None, follow_links=True): imports = set() raw_imports = set() candidates = [] @@ -78,7 +84,8 @@ def get_all_imports(path, encoding=None, extra_ignore_dirs=None, follow_links=Tr candidates += [os.path.splitext(fn)[0] for fn in files] for file_name in files: - with open_func(os.path.join(root, file_name), "r", encoding=encoding) as f: + file_name = os.path.join(root, file_name) + with open_func(file_name, "r", encoding=encoding) as f: contents = f.read() try: tree = ast.parse(contents) @@ -91,19 +98,19 @@ def get_all_imports(path, encoding=None, extra_ignore_dirs=None, follow_links=Tr except Exception as exc: if ignore_errors: traceback.print_exc(exc) - logging.warn("Failed on file: %s" % os.path.join(root, file_name)) + logging.warn("Failed on file: %s" % file_name) continue else: - logging.error("Failed on file: %s" % os.path.join(root, file_name)) + logging.error("Failed on file: %s" % file_name) raise exc - - # Clean up imports for name in [n for n in raw_imports if n]: - # Sanity check: Name could have been None if the import statement was as from . import X + # Sanity check: Name could have been None if the import + # statement was as ``from . import X`` # Cleanup: We only want to first part of the import. - # Ex: from django.conf --> django.conf. But we only want django as an import + # Ex: from django.conf --> django.conf. But we only want django + # as an import. cleaned_name, _, _ = name.partition('.') imports.add(cleaned_name) @@ -128,8 +135,10 @@ def generate_requirements_file(path, imports): imports=", ".join([x['name'] for x in imports]) )) fmt = '{name}=={version}' - out_file.write('\n'.join(fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports) + '\n') + out_file.write('\n'.join( + fmt.format(**item) if item['version'] else '{name}'.format(**item) + for item in imports) + '\n') + def output_requirements(imports): logging.debug('Writing {num} requirements: {imports} to stdout'.format( @@ -137,16 +146,19 @@ def output_requirements(imports): imports=", ".join([x['name'] for x in imports]) )) fmt = '{name}=={version}' - print('\n'.join(fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports)) + print('\n'.join( + fmt.format(**item) if item['version'] else '{name}'.format(**item) + for item in imports)) -def get_imports_info(imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): +def get_imports_info( + imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): result = [] for item in imports: try: - response = requests.get("{0}{1}/json".format(pypi_server, item), proxies=proxy) + response = requests.get( + "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: if hasattr(response.content, 'decode'): data = json2package(response.content.decode()) @@ -170,11 +182,13 @@ def get_locally_installed_packages(encoding=None): for root, dirs, files in os.walk(path): for item in files: if "top_level" in item: - with open_func(os.path.join(root, item), "r", encoding=encoding) as f: + item = os.path.join(root, item) + with open_func(item, "r", encoding=encoding) as f: package = root.split(os.sep)[-1].split("-") try: package_import = f.read().strip().split("\n") - except: + except: # NOQA + # TODO: What errors do we intend to suppress here? continue for i_item in package_import: if ((i_item not in ignore) and @@ -235,6 +249,7 @@ def get_name_without_alias(name): def join(f): return os.path.join(os.path.dirname(__file__), f) + def parse_requirements(file_): """Parse a requirements formatted file. @@ -252,7 +267,9 @@ def parse_requirements(file_): tuple: The contents of the file, excluding comments. """ modules = [] - delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar + # For the dependency identifier specification, see + # https://www.python.org/dev/peps/pep-0508/#complete-grammar + delim = ["<", ">", "=", "!", "~"] try: f = open_func(file_, "r") @@ -267,7 +284,8 @@ def parse_requirements(file_): data = [x for x in data if x[0].isalpha()] for x in data: - if not any([y in x for y in delim]): # Check for modules w/o a specifier. + # Check for modules w/o a specifier. + if not any([y in x for y in delim]): modules.append({"name": x, "version": None}) for y in x: if y in delim: @@ -305,11 +323,13 @@ def compare_modules(file_, imports): def diff(file_, imports): - """Display the difference between modules in a file and imported modules.""" + """Display the difference between modules in a file and imported modules.""" # NOQA modules_not_imported = compare_modules(file_, imports) - logging.info("The following modules are in {} but do not seem to be imported: " - "{}".format(file_, ", ".join(x for x in modules_not_imported))) + logging.info( + "The following modules are in {} but do not seem to be imported: " + "{}".format(file_, ", ".join(x for x in modules_not_imported))) + def clean(file_, imports): """Remove modules that aren't imported in project from file.""" @@ -336,6 +356,7 @@ def clean(file_, imports): logging.info("Successfully cleaned up requirements in " + file_) + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -386,7 +407,10 @@ def init(args): clean(args["--clean"], imports) return - if not args["--print"] and not args["--savepath"] and not args["--force"] and os.path.exists(path): + if (not args["--print"] + and not args["--savepath"] + and not args["--force"] + and os.path.exists(path)): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return From 4b2ad2dc41fa054ffb71a0b40c602f6b3a6fb3db Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Tue, 24 Oct 2017 16:46:55 -0400 Subject: [PATCH 6/9] [WIP] Consolidate logic for writing to a file and to stdout --- pipreqs/pipreqs.py | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index d21b071..c0f9c55 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -33,6 +33,7 @@ Options: that are not imported in project. """ from __future__ import print_function, absolute_import +from contextlib import contextmanager import os import sys import re @@ -61,6 +62,37 @@ else: py2_exclude = ["concurrent", "concurrent.futures"] +@contextmanager +def _open(filename=None, mode='r'): + """Open a file or ``sys.stdout`` depending on the provided filename. + + Args: + filename (str): The path to the file that should be opened. If + ``None`` or ``'-'``, ``sys.stdout`` or ``sys.stdin`` is + returned depending on the desired mode. Defaults to ``None``. + mode (str): The mode that should be used to open the file. + + Yields: + A file handle. + + """ + if not filename or filename == '-': + if not mode or 'r' in mode: + file = sys.stdin + elif 'w' in mode: + file = sys.stdout + else: + raise ValueError('Invalid mode for file: {}'.format(mode)) + else: + file = open(filename, mode) + + try: + yield file + finally: + if file not in (sys.stdin, sys.stdout): + file.close() + + def get_all_imports( path, encoding=None, extra_ignore_dirs=None, follow_links=True): imports = set() @@ -128,7 +160,7 @@ def filter_line(l): def generate_requirements_file(path, imports): - with open(path, "w") as out_file: + with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, @@ -141,14 +173,7 @@ def generate_requirements_file(path, imports): def output_requirements(imports): - logging.debug('Writing {num} requirements: {imports} to stdout'.format( - num=len(imports), - imports=", ".join([x['name'] for x in imports]) - )) - fmt = '{name}=={version}' - print('\n'.join( - fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports)) + generate_requirements_file('-', imports) def get_imports_info( From 84b2e37707f0c145e78bcc4442e77a2591fc3dfc Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Thu, 26 Oct 2017 12:16:32 -0400 Subject: [PATCH 7/9] Simplify get_pkg_names function - Hoist non-file-reading logic outside of the file context manager - Use a dict instead of a list for faster / more Pythonic lookups - Use a set to simplify the add / append logic - Move import sorting from `get_all_imports` to `get_pkg_names` for to account for set ordering. This change may also affect #89. - Add a docstring --- pipreqs/pipreqs.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index d21b071..5cf763e 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -120,7 +120,7 @@ def get_all_imports( with open(join("stdlib"), "r") as f: data = [x.strip() for x in f.readlines()] data = [x for x in data if x not in py2_exclude] if py2 else data - return sorted(list(set(packages) - set(data))) + return list(set(packages) - set(data)) def filter_line(l): @@ -224,18 +224,24 @@ def get_import_local(imports, encoding=None): def get_pkg_names(pkgs): - result = [] + """Get PyPI package names from a list of imports. + + Args: + pkgs (List[str]): List of import names. + + Returns: + List[str]: The corresponding PyPI package names. + + """ + result = set() with open(join("mapping"), "r") as f: - data = [x.strip().split(":") for x in f.readlines()] - for pkg in pkgs: - toappend = pkg - for item in data: - if item[0] == pkg: - toappend = item[1] - break - if toappend not in result: - result.append(toappend) - return result + data = dict(x.strip().split(":") for x in f) + for pkg in pkgs: + # Look up the mapped requirement. If a mapping isn't found, + # simply use the package name. + result.add(data.get(pkg, pkg)) + # Return a sorted list for backward compatibility. + return sorted(result) def get_name_without_alias(name): From aae6c61f090201f2f29be95c47ce1b9b3a09e8e4 Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Fri, 20 Oct 2017 23:43:45 -0400 Subject: [PATCH 8/9] Clean up set and file usage in get_all_imports - Move logic that doesn't need to be inside of file context managers outside - Remove redundant `set` and `list` function calls --- pipreqs/pipreqs.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 5cf763e..9a466f3 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -87,22 +87,22 @@ def get_all_imports( file_name = os.path.join(root, file_name) with open_func(file_name, "r", encoding=encoding) as f: contents = f.read() - try: - tree = ast.parse(contents) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for subnode in node.names: - raw_imports.add(subnode.name) - elif isinstance(node, ast.ImportFrom): - raw_imports.add(node.module) - except Exception as exc: - if ignore_errors: - traceback.print_exc(exc) - logging.warn("Failed on file: %s" % file_name) - continue - else: - logging.error("Failed on file: %s" % file_name) - raise exc + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for subnode in node.names: + raw_imports.add(subnode.name) + elif isinstance(node, ast.ImportFrom): + raw_imports.add(node.module) + except Exception as exc: + if ignore_errors: + traceback.print_exc(exc) + logging.warn("Failed on file: %s" % file_name) + continue + else: + logging.error("Failed on file: %s" % file_name) + raise exc # Clean up imports for name in [n for n in raw_imports if n]: @@ -114,13 +114,14 @@ def get_all_imports( cleaned_name, _, _ = name.partition('.') imports.add(cleaned_name) - packages = set(imports) - set(set(candidates) & set(imports)) + packages = imports - (set(candidates) & imports) logging.debug('Found packages: {0}'.format(packages)) with open(join("stdlib"), "r") as f: - data = [x.strip() for x in f.readlines()] - data = [x for x in data if x not in py2_exclude] if py2 else data - return list(set(packages) - set(data)) + data = {x.strip() for x in f} + + data = {x for x in data if x not in py2_exclude} if py2 else data + return list(packages - data) def filter_line(l): From 52505ce32c692ecb79ead8dfb30369689555d234 Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Tue, 31 Oct 2017 15:18:45 -0400 Subject: [PATCH 9/9] Add dates to recent changelog entries Via https://pypi.python.org/pypi/pipreqs/json --- HISTORY.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b0126ab..b9d17e7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,19 +3,19 @@ History ------- -0.4.8 +0.4.8 (2017-06-30) -------------------- * Implement '--clean' and '--diff' (kxrd) * Exclude concurrent{,.futures} from stdlib if py2 (kxrd) -0.4.7 +0.4.7 (2017-04-20) -------------------- * BUG: remove package/version duplicates * Style: pep8 -0.4.5 +0.4.5 (2016-12-13) --------------------- * Fixed the --pypi-server option