github/pipreqs
1
1
Fork 0
mirror of https://github.com/bndr/pipreqs.git synced 2025-06-01 09:00:15 +00:00
Code Issues 3 Packages Projects Releases Wiki Activity

Revert "Revert all commits since last release"

Browse Source 
This reverts commit 90102acdbb23c09574d27df8bd1f568d34e0cfd3.
Now that we are ready to make a new release we can revert the revert and
hopefuly never have to solve a mess like this again to keep master
synchronized with the latest release
This commit is contained in:
Alan Barzilay 2021-09-04 15:12:27 -03:00
parent 90102acdbb
commit 8001001e9c
11 changed files with 300 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -12,13 +12,13 @@ matrix:
env: TOX_ENV=py34
- python: 2.7
env: TOX_ENV=py27
- python: pypy
env: TOX_ENV=pypy
- python: pypy3
env: TOX_ENV=pypy3
- python: 3.6
env: TOX_ENV=flake8
# Use tox to run tests on Travis-CI to keep one unified method of running tests in any environment
install:
install:
- pip install coverage coveralls tox
# Command to run tests, e.g. python setup.py test

3
AUTHORS.rst
View File

@ -10,4 +10,5 @@ Development Lead
Contributors
------------
None yet. Why not be the first?
* Jake Teo <mapattacker@gmail.com>
* Jerome Chan <cjerome94@gmail.com>

5
HISTORY.rst
View File

@ -3,6 +3,11 @@
History
-------
0.4.11 (2020-03-29)
--------------------
* Implement '--mode' (Jake Teo, Jerome Chan)
0.4.8 (2017-06-30)
--------------------

9
README.rst
View File

@ -49,7 +49,10 @@ Usage
--force Overwrite existing requirements.txt
--diff <file> Compare modules in requirements.txt to project imports.
--clean <file> Clean up requirements.txt by removing modules that are not imported in project.
--no-pin Omit version of output packages.
--mode <scheme> Enables dynamic versioning with <compat>, <gt> or <non-pin> schemes.
<compat> | e.g. Flask~=1.1.2
<gt> | e.g. Flask>=1.1.2
<no-pin> | e.g. Flask
Example
-------
@ -70,5 +73,5 @@ Why not pip freeze?
-------------------
- ``pip freeze`` only saves the packages that are installed with ``pip install`` in your environment.
- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project. (if you don't have virtualenv)
- and sometimes you just need to create requirements.txt for a new project without installing modules.
- ``pip freeze`` saves all packages in the environment including those that you don't use in your current project (if you don't have ``virtualenv``).
- and sometimes you just need to create ``requirements.txt`` for a new project without installing modules.

10
pipreqs/mapping
View File

@ -1,3 +1,4 @@
AFQ:pyAFQ
AG_fft_tools:agpy
ANSI:pexpect
Adafruit:Adafruit_Libraries
@ -13,6 +14,7 @@ Crypto:pycryptodome
Cryptodome:pycryptodomex
FSM:pexpect
FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper
functional:pyfunctional
GeoBaseMain:GeoBasesDev
GeoBases:GeoBasesDev
Globals:Zope2
@ -22,6 +24,7 @@ Kittens:astro_kittens
Levenshtein:python_Levenshtein
Lifetime:Zope2
MethodObject:ExtensionClass
MySQLdb:MySQL-python
OFS:Zope2
OpenGL:PyOpenGL
OpenSSL:pyOpenSSL
@ -592,6 +595,7 @@ devtools:tg.devtools
dgis:2gis
dhtmlparser:pyDHTMLParser
digitalocean:python_digitalocean
discord:discord.py
distribute_setup:ez_setup
distutils2:Distutils2
django:Django
@ -675,6 +679,7 @@ geventwebsocket:gevent_websocket
gflags:python_gflags
git:GitPython
github:PyGithub
github3:github3.py
gitpy:git_py
globusonline:globusonline_transfer_api_client
google:protobuf
@ -701,7 +706,7 @@ html:pies2overrides
htmloutput:nosehtmloutput
http:pies2overrides
hvad:django_hvad
krbV:krbv
hydra:hydra-core
i99fix:199Fix
igraph:python_igraph
imdb:IMDbPY
@ -727,6 +732,7 @@ keyczar:python_keyczar
keyedcache:django_keyedcache
keystoneclient:python_keystoneclient
kickstarter:kickstart
krbv:krbV
kss:kss.core
kuyruk:Kuyruk
langconv:AdvancedLangConv
@ -798,7 +804,6 @@ msgpack:msgpack_python
mutations:aino_mutations
mws:amazon_mws
mysql:mysql_connector_repackaged
MySQL-python:MySQLdb
native_tags:django_native_tags
ndg:ndg_httpsclient
nereid:trytond_nereid
@ -999,6 +1004,7 @@ ruamel:ruamel.base
s2repoze:pysaml2
saga:saga_python
saml2:pysaml2
samtranslator:aws-sam-translator
sass:libsass
sassc:libsass
sasstests:libsass

54
pipreqs/pipreqs.py Executable file → Normal file
View File

@ -31,7 +31,11 @@ Options:
imports.
--clean <file> Clean up requirements.txt by removing modules
that are not imported in project.
--no-pin Omit version of output packages.
--mode <scheme> Enables dynamic versioning with <compat>,
<gt> or <non-pin> schemes.
<compat> | e.g. Flask~=1.1.2
<gt> | e.g. Flask>=1.1.2
<no-pin> | e.g. Flask
"""
from __future__ import print_function, absolute_import
from contextlib import contextmanager
@ -157,25 +161,25 @@ def get_all_imports(
return list(packages - data)
def filter_line(l):
return len(l) > 0 and l[0] != "#"
def filter_line(line):
return len(line) > 0 and line[0] != "#"
def generate_requirements_file(path, imports):
def generate_requirements_file(path, imports, symbol):
with _open(path, "w") as out_file:
logging.debug('Writing {num} requirements: {imports} to {file}'.format(
num=len(imports),
file=path,
imports=", ".join([x['name'] for x in imports])
))
fmt = '{name}=={version}'
fmt = '{name}' + symbol + '{version}'
out_file.write('\n'.join(
fmt.format(**item) if item['version'] else '{name}'.format(**item)
for item in imports) + '\n')
def output_requirements(imports):
generate_requirements_file('-', imports)
def output_requirements(imports, symbol):
generate_requirements_file('-', imports, symbol)
def get_imports_info(
@ -368,6 +372,11 @@ def diff(file_, imports):
def clean(file_, imports):
"""Remove modules that aren't imported in project from file."""
modules_not_imported = compare_modules(file_, imports)
if len(modules_not_imported) == 0:
logging.info("Nothing to clean in " + file_)
return
re_remove = re.compile("|".join(modules_not_imported))
to_write = []
@ -392,6 +401,18 @@ def clean(file_, imports):
logging.info("Successfully cleaned up requirements in " + file_)
def dynamic_versioning(scheme, imports):
"""Enables dynamic versioning with <compat>, <gt> or <non-pin> schemes."""
if scheme == "no-pin":
imports = [{"name": item["name"], "version": ""} for item in imports]
symbol = ""
elif scheme == "gt":
symbol = ">="
elif scheme == "compat":
symbol = "~="
return imports, symbol
def init(args):
encoding = args.get('--encoding')
extra_ignore_dirs = args.get('--ignore')
@ -430,6 +451,8 @@ def init(args):
imports = local + get_imports_info(difference,
proxy=proxy,
pypi_server=pypi_server)
# sort imports based on lowercase name of package, similar to `pip freeze`.
imports = sorted(imports, key=lambda x: x['name'].lower())
path = (args["--savepath"] if args["--savepath"] else
os.path.join(input_path, "requirements.txt"))
@ -446,18 +469,25 @@ def init(args):
and not args["--savepath"]
and not args["--force"]
and os.path.exists(path)):
logging.warning("Requirements.txt already exists, "
logging.warning("requirements.txt already exists, "
"use --force to overwrite it")
return
if args.get('--no-pin'):
imports = [{'name': item["name"], 'version': ''} for item in imports]
if args["--mode"]:
scheme = args.get("--mode")
if scheme in ["compat", "gt", "no-pin"]:
imports, symbol = dynamic_versioning(scheme, imports)
else:
raise ValueError("Invalid argument for mode flag, "
"use 'compat', 'gt' or 'no-pin' instead")
else:
symbol = "=="
if args["--print"]:
output_requirements(imports)
output_requirements(imports, symbol)
logging.info("Successfully output requirements")
else:
generate_requirements_file(path, imports)
generate_requirements_file(path, imports, symbol)
logging.info("Successfully saved requirements file in " + path)

2
pipreqs/stdlib
View File

@ -133,6 +133,7 @@ curses
curses.ascii
curses.panel
curses.textpad
dataclasses
datetime
dbhash
dbm
@ -370,6 +371,7 @@ robotparser
runpy
sched
ScrolledText
secrets
select
selectors
sets

4
tests/_data/test.py
View File

@ -31,6 +31,10 @@ from pyflakes.test.test_imports import Test as TestImports
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# see issue #88
import analytics
import flask_seasurf
import atexit
from __future__ import print_function
from docopt import docopt

65
tests/_data_clean/test.py Normal file
View File

@ -0,0 +1,65 @@
"""unused import"""
# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
import xml.etree # [unused-import]
import xml.sax # [unused-import]
import os.path as test # [unused-import]
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# +1:[unused-import,unused-import]
from collections import deque, OrderedDict, Counter
# All imports above should be ignored
import requests # [unused-import]
# setuptools
import zipimport # command/easy_install.py
# twisted
from importlib import invalidate_caches # python/test/test_deprecate.py
# astroid
import zipimport # manager.py
# IPython
from importlib.machinery import all_suffixes # core/completerlib.py
import importlib # html/notebookapp.py
from IPython.utils.importstring import import_item # Many files
# pyflakes
# test/test_doctests.py
from pyflakes.test.test_imports import Test as TestImports
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# see issue #88
import analytics
import flask_seasurf
import atexit
from __future__ import print_function
from docopt import docopt
import curses, logging, sqlite3
import logging
import os
import sqlite3
import time
import sys
import signal
import bs4
import nonexistendmodule
import boto as b, peewee as p
# import django
import flask.ext.somext # # #
# from sqlalchemy import model
try:
import ujson as json
except ImportError:
import json
import models
def main():
pass
import after_method_is_valid_even_if_not_pep8

202
tests/test_pipreqs.py Executable file → Normal file
View File

@ -18,22 +18,40 @@ from pipreqs import pipreqs
class TestPipreqs(unittest.TestCase):
def setUp(self):
self.modules = ['flask', 'requests', 'sqlalchemy',
'docopt', 'boto', 'ipython', 'pyflakes', 'nose',
'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ]
self.modules = [
'flask', 'requests', 'sqlalchemy', 'docopt', 'boto', 'ipython',
'pyflakes', 'nose', 'analytics', 'flask_seasurf', 'peewee',
'ujson', 'nonexistendmodule', 'bs4',
'after_method_is_valid_even_if_not_pep8'
]
self.modules2 = ['beautifulsoup4']
self.local = ["docopt", "requests", "nose", 'pyflakes']
self.project = os.path.join(os.path.dirname(__file__), "_data")
self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data")
self.project_with_ignore_directory = os.path.join(os.path.dirname(__file__), "_data_ignore")
self.project_with_duplicated_deps = os.path.join(os.path.dirname(__file__), "_data_duplicated_deps")
self.project_clean = os.path.join(
os.path.dirname(__file__),
"_data_clean"
)
self.project_invalid = os.path.join(
os.path.dirname(__file__),
"_invalid_data"
)
self.project_with_ignore_directory = os.path.join(
os.path.dirname(__file__),
"_data_ignore"
)
self.project_with_duplicated_deps = os.path.join(
os.path.dirname(__file__),
"_data_duplicated_deps"
)
self.requirements_path = os.path.join(self.project, "requirements.txt")
self.alt_requirement_path = os.path.join(
self.project, "requirements2.txt")
self.project,
"requirements2.txt"
)
def test_get_all_imports(self):
imports = pipreqs.get_all_imports(self.project)
self.assertEqual(len(imports), 13)
self.assertEqual(len(imports), 15)
for item in imports:
self.assertTrue(
item.lower() in self.modules, "Import is missing: " + item)
@ -54,7 +72,8 @@ class TestPipreqs(unittest.TestCase):
"""
Test that invalid python files cannot be imported.
"""
self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
self.assertRaises(
SyntaxError, pipreqs.get_all_imports, self.project_invalid)
def test_get_imports_info(self):
"""
@ -62,8 +81,9 @@ class TestPipreqs(unittest.TestCase):
"""
imports = pipreqs.get_all_imports(self.project)
with_info = pipreqs.get_imports_info(imports)
# Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8"
self.assertEqual(len(with_info), 11)
# Should contain 10 items without the "nonexistendmodule" and
# "after_method_is_valid_even_if_not_pep8"
self.assertEqual(len(with_info), 13)
for item in with_info:
self.assertTrue(
item['name'].lower() in self.modules,
@ -77,10 +97,12 @@ class TestPipreqs(unittest.TestCase):
def test_get_use_local_only(self):
"""
Test without checking PyPI, check to see if names of local imports matches what we expect
Test without checking PyPI, check to see if names of local
imports matches what we expect
- Note even though pyflakes isn't in requirements.txt,
It's added to locals since it is a development dependency for testing
It's added to locals since it is a development dependency
for testing
"""
# should find only docopt and requests
imports_with_info = pipreqs.get_import_local(self.modules)
@ -89,24 +111,28 @@ class TestPipreqs(unittest.TestCase):
def test_init(self):
"""
Test that all modules we will test upon, are in requirements file
Test that all modules we will test upon are in requirements file
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None})
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
for item in self.modules[:-3]:
self.assertTrue(item.lower() in data)
# It should be sorted based on names.
data = data.strip().split('\n')
self.assertEqual(data, sorted(data))
def test_init_local_only(self):
"""
Test that items listed in requirements.text are the same as locals expected
Test that items listed in requirements.text are the same
as locals expected
"""
pipreqs.init({'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None,
'--diff': None, '--clean': None})
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.readlines()
@ -116,11 +142,12 @@ class TestPipreqs(unittest.TestCase):
def test_init_savepath(self):
"""
Test that we can save requiremnts.tt correctly to a different path
Test that we can save requirements.txt correctly
to a different path
"""
pipreqs.init({'<path>': self.project, '--savepath':
self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False,
"--diff": None, "--clean": None})
pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path,
'--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.alt_requirement_path) == 1
with open(self.alt_requirement_path, "r") as f:
data = f.read().lower()
@ -131,13 +158,14 @@ class TestPipreqs(unittest.TestCase):
def test_init_overwrite(self):
"""
Test that if requiremnts.txt exists, it will not automatically be overwritten
Test that if requiremnts.txt exists, it will not be
automatically overwritten
"""
with open(self.requirements_path, "w") as f:
f.write("should_not_be_overwritten")
pipreqs.init({'<path>': self.project, '--savepath': None,
'--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
"--diff": None, "--clean": None})
pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None,
'--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
@ -145,35 +173,43 @@ class TestPipreqs(unittest.TestCase):
def test_get_import_name_without_alias(self):
"""
Test that function get_name_without_alias() will work on a string.
- Note: This isn't truly needed when pipreqs is walking the AST to find imports
Test that function get_name_without_alias()
will work on a string.
- Note: This isn't truly needed when pipreqs is walking
the AST to find imports
"""
import_name_with_alias = "requests as R"
expected_import_name_without_alias = "requests"
import_name_without_aliases = pipreqs.get_name_without_alias(
import_name_with_alias)
self.assertEqual(
import_name_without_aliases, expected_import_name_without_alias)
import_name_without_aliases,
expected_import_name_without_alias
)
def test_custom_pypi_server(self):
"""
Test that trying to get a custom pypi sever fails correctly
"""
self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'})
self.assertRaises(
requests.exceptions.MissingSchema, pipreqs.init,
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': 'nonexistent'}
)
def test_ignored_directory(self):
"""
Test --ignore parameter
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy':None,
'--pypi-server':None,
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy':None, '--pypi-server':None,
'--ignore':'.ignored_dir,.ignore_second',
'--diff': None,
'--clean': None
'--clean': None,
'--mode': None
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
@ -181,9 +217,27 @@ class TestPipreqs(unittest.TestCase):
for item in ['click', 'getpass']:
self.assertFalse(item.lower() in data)
def test_omit_version(self):
def test_dynamic_version_no_pin_scheme(self):
"""
Test --no-pin parameter
Test --mode=no-pin
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy': None, '--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'no-pin'
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.read().lower()
for item in ['beautifulsoup4', 'boto']:
self.assertTrue(item.lower() in data)
def test_dynamic_version_gt_scheme(self):
"""
Test --mode=gt
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
@ -192,13 +246,81 @@ class TestPipreqs(unittest.TestCase):
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--no-pin': True
'--mode': 'gt'
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
for item in data:
symbol = '>='
message = 'symbol is not in item'
self.assertIn(symbol, item, message)
def test_dynamic_version_compat_scheme(self):
"""
Test --mode=compat
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None, '--print': False,
'--use-local': None, '--force': True,
'--proxy': None,
'--pypi-server': None,
'--diff': None,
'--clean': None,
'--mode': 'compat'
}
)
with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f:
data = f.readlines()
for item in data:
symbol = '~='
message = 'symbol is not in item'
self.assertIn(symbol, item, message)
def test_clean(self):
"""
Test --clean parameter
"""
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
assert os.path.exists(self.requirements_path) == 1
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': None, '--proxy': None,
'--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']:
self.assertFalse(item.lower() in data)
for item in self.modules[:-3]:
self.assertTrue(item.lower() in data)
def test_clean_with_imports_to_clean(self):
"""
Test --clean parameter when there are imports to clean
"""
cleaned_module = 'sqlalchemy'
pipreqs.init(
{'<path>': self.project, '--savepath': None, '--print': False,
'--use-local': None, '--force': True, '--proxy': None,
'--pypi-server': None, '--diff': None, '--clean': None,
'--mode': None}
)
assert os.path.exists(self.requirements_path) == 1
modules_clean = [m for m in self.modules if m != cleaned_module]
pipreqs.init(
{'<path>': self.project_clean, '--savepath': None,
'--print': False, '--use-local': None, '--force': None,
'--proxy': None, '--pypi-server': None, '--diff': None,
'--clean': self.requirements_path, '--mode': 'non-pin'}
)
with open(self.requirements_path, "r") as f:
data = f.read().lower()
self.assertTrue(cleaned_module not in data)
def tearDown(self):
"""

2
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = py27, py34, py35, py36, pypy, flake8
envlist = py27, py34, py35, py36, pypy3, flake8
[testenv]
setenv =