github/pipreqs
1
1
Fork 0
mirror of https://github.com/bndr/pipreqs.git synced 2025-06-06 03:25:21 +00:00
Code Issues 3 Packages Projects Releases Wiki Activity

Merge pull request #37 from 9cloud/AST-Walking

Browse Source 
Walk Abstract Syntax Tree to find imports
This commit is contained in:
Vadim Kravcenko 2016-01-28 22:03:47 +01:00
commit 9c39a1a3ed
4 changed files with 79 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
pipreqs/pipreqs.py
View File

@ -23,7 +23,8 @@ import sys
import re
import logging
import codecs
import ast
import traceback
from docopt import docopt
import requests
from yarg import json2package
@ -43,8 +44,10 @@ else:
def get_all_imports(path, encoding=None):
imports = []
imports = set()
raw_imports = set()
candidates = []
ignore_errors = False
ignore_dirs = [".hg", ".svn", ".git", "__pycache__", "env", "venv"]
for root, dirs, files in os.walk(path):
@ -56,19 +59,33 @@ def get_all_imports(path, encoding=None):
candidates += [os.path.splitext(fn)[0] for fn in files]
for file_name in files:
with open_func(os.path.join(root, file_name), "r", encoding=encoding) as f:
contents = re.sub(re.compile("'''.+?'''", re.DOTALL), '', f.read())
contents = re.sub(re.compile('""".+?"""', re.DOTALL), "", contents)
lines = contents.split("\n")
lines = filter(
filter_line, map(lambda l: l.partition("#")[0].strip(), lines))
for line in lines:
if "(" in line:
break
for rex in REGEXP:
s = rex.findall(line)
for item in s:
res = map(get_name_without_alias, item.split(","))
imports = imports + [x for x in res if len(x) > 0]
contents = f.read()
try:
tree = ast.parse(contents)
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for subnode in node.names:
raw_imports.add(subnode.name)
elif isinstance(node, ast.ImportFrom):
raw_imports.add(node.module)
except Exception as exc:
if ignore_errors:
traceback.print_exc(exc)
logging.warn("Failed on file: %s" % os.path.join(root, file_name))
continue
else:
logging.error("Failed on file: %s" % os.path.join(root, file_name))
raise exc
# Clean up imports
for name in [n for n in raw_imports if n]:
# Sanity check: Name could have been None if the import statement was as from . import X
# Cleanup: We only want to first part of the import.
# Ex: from django.conf --> django.conf. But we only want django as an import
cleaned_name, _, _ = name.partition('.')
imports.add(cleaned_name)
packages = set(imports) - set(set(candidates) & set(imports))
logging.debug('Found packages: {0}'.format(packages))

4
tests/_data/test.py
View File

@ -43,7 +43,7 @@ import sys
import signal
import bs4
import nonexistendmodule
import boto as b, import peewee as p,
import boto as b, peewee as p
# import django
import flask.ext.somext # # #
from sqlalchemy import model
@ -58,4 +58,4 @@ import models
def main():
pass
import after_method_should_be_ignored
import after_method_is_valid_even_if_not_pep8

1
tests/_invalid_data/invalid.py Normal file
View File

@ -0,0 +1 @@
import boto as b, import peewee as p,

50
tests/test_pipreqs.py
View File

@ -20,17 +20,18 @@ class TestPipreqs(unittest.TestCase):
def setUp(self):
self.modules = ['flask', 'requests', 'sqlalchemy',
'docopt', 'boto', 'ipython', 'pyflakes', 'nose',
'peewee', 'ujson', 'nonexistendmodule', 'bs4', ]
'peewee', 'ujson', 'nonexistendmodule', 'bs4', 'after_method_is_valid_even_if_not_pep8' ]
self.modules2 = ['beautifulsoup4']
self.local = ["docopt", "requests", "nose"]
self.local = ["docopt", "requests", "nose", 'pyflakes']
self.project = os.path.join(os.path.dirname(__file__), "_data")
self.project_invalid = os.path.join(os.path.dirname(__file__), "_invalid_data")
self.requirements_path = os.path.join(self.project, "requirements.txt")
self.alt_requirement_path = os.path.join(
self.project, "requirements2.txt")
def test_get_all_imports(self):
imports = pipreqs.get_all_imports(self.project)
self.assertEqual(len(imports), 12)
self.assertEqual(len(imports), 13)
for item in imports:
self.assertTrue(
item.lower() in self.modules, "Import is missing: " + item)
@ -41,10 +42,19 @@ class TestPipreqs(unittest.TestCase):
self.assertFalse("django" in imports)
self.assertFalse("models" in imports)
def test_invalid_python(self):
"""
Test that invalid python files cannot be imported.
"""
self.assertRaises(SyntaxError, pipreqs.get_all_imports, self.project_invalid)
def test_get_imports_info(self):
"""
Test to see that the right number of packages were found on PyPI
"""
imports = pipreqs.get_all_imports(self.project)
with_info = pipreqs.get_imports_info(imports)
# Should contain only 5 Elements without the "nonexistendmodule"
# Should contain 10 items without the "nonexistendmodule" and "after_method_is_valid_even_if_not_pep8"
self.assertEqual(len(with_info), 10)
for item in with_info:
self.assertTrue(
@ -52,21 +62,33 @@ class TestPipreqs(unittest.TestCase):
"Import item appears to be missing " + item['name'])
def test_get_use_local_only(self):
"""
Test without checking PyPI, check to see if names of local imports matches what we expect
- Note even though pyflakes isn't in requirements.txt,
It's added to locals since it is a development dependency for testing
"""
# should find only docopt and requests
imports_with_info = pipreqs.get_import_local(self.modules)
for item in imports_with_info:
self.assertTrue(item['name'].lower() in self.local)
def test_init(self):
"""
Test that all modules we will test upon, are in requirements file
"""
pipreqs.init({'<path>': self.project, '--savepath': None,
'--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None})
assert os.path.exists(self.requirements_path) == 1
with open(self.requirements_path, "r") as f:
data = f.read().lower()
for item in self.modules[:-2]:
for item in self.modules[:-3]:
self.assertTrue(item.lower() in data)
def test_init_local_only(self):
"""
Test that items listed in requirements.text are the same as locals expected
"""
pipreqs.init({'<path>': self.project, '--savepath': None,
'--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None})
assert os.path.exists(self.requirements_path) == 1
@ -77,17 +99,23 @@ class TestPipreqs(unittest.TestCase):
self.assertTrue(item[0].lower() in self.local)
def test_init_savepath(self):
"""
Test that we can save requiremnts.tt correctly to a different path
"""
pipreqs.init({'<path>': self.project, '--savepath':
self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None})
assert os.path.exists(self.alt_requirement_path) == 1
with open(self.alt_requirement_path, "r") as f:
data = f.read().lower()
for item in self.modules[:-2]:
for item in self.modules[:-3]:
self.assertTrue(item.lower() in data)
for item in self.modules2:
self.assertTrue(item.lower() in data)
def test_init_overwrite(self):
"""
Test that if requiremnts.txt exists, it will not automatically be overwritten
"""
with open(self.requirements_path, "w") as f:
f.write("should_not_be_overwritten")
pipreqs.init({'<path>': self.project, '--savepath': None,
@ -98,6 +126,10 @@ class TestPipreqs(unittest.TestCase):
self.assertEqual(data, "should_not_be_overwritten")
def test_get_import_name_without_alias(self):
"""
Test that function get_name_without_alias() will work on a string.
- Note: This isn't truly needed when pipreqs is walking the AST to find imports
"""
import_name_with_alias = "requests as R"
expected_import_name_without_alias = "requests"
import_name_without_aliases = pipreqs.get_name_without_alias(
@ -106,10 +138,16 @@ class TestPipreqs(unittest.TestCase):
import_name_without_aliases, expected_import_name_without_alias)
def test_custom_pypi_server(self):
"""
Test that trying to get a custom pypi sever fails correctly
"""
self.assertRaises(requests.exceptions.MissingSchema, pipreqs.init, {'<path>': self.project, '--savepath': None,
'--use-local': None, '--force': True, '--proxy': None, '--pypi-server': 'nonexistent'})
def tearDown(self):
"""
Remove requiremnts.txt files that were written
"""
try:
os.remove(self.requirements_path)
except OSError: