black auto format and sort imports

This commit is contained in:
c-w-m 2023-01-12 09:50:00 -07:00
parent e0830d44a5
commit ffc2c11d99
6 changed files with 214 additions and 192 deletions

View File

@ -38,28 +38,29 @@ Options:
<gt> | e.g. Flask>=1.1.2
<no-pin> | e.g. Flask
"""
from contextlib import contextmanager
import os
import sys
import re
import logging
import ast
import logging
import os
import re
import sys
import traceback
from docopt import docopt
from contextlib import contextmanager
import requests
from docopt import docopt
from yarg import json2package
from yarg.exceptions import HTTPError
from pipreqs import __version__
REGEXP = [
re.compile(r'^import (.+)$'),
re.compile(r'^from ((?!\.+).*?) import (?:.*)$')
re.compile(r"^import (.+)$"),
re.compile(r"^from ((?!\.+).*?) import (?:.*)$"),
]
@contextmanager
def _open(filename=None, mode='r'):
def _open(filename=None, mode="r"):
"""Open a file or ``sys.stdout`` depending on the provided filename.
Args:
@ -72,13 +73,13 @@ def _open(filename=None, mode='r'):
A file handle.
"""
if not filename or filename == '-':
if not mode or 'r' in mode:
if not filename or filename == "-":
if not mode or "r" in mode:
file = sys.stdin
elif 'w' in mode:
elif "w" in mode:
file = sys.stdout
else:
raise ValueError('Invalid mode for file: {}'.format(mode))
raise ValueError("Invalid mode for file: {}".format(mode))
else:
file = open(filename, mode)
@ -90,7 +91,8 @@ def _open(filename=None, mode='r'):
def get_all_imports(
path, encoding=None, extra_ignore_dirs=None, follow_links=True):
path, encoding=None, extra_ignore_dirs=None, follow_links=True
):
imports = set()
raw_imports = set()
candidates = []
@ -139,11 +141,11 @@ def get_all_imports(
# Cleanup: We only want to first part of the import.
# Ex: from django.conf --> django.conf. But we only want django
# as an import.
cleaned_name, _, _ = name.partition('.')
cleaned_name, _, _ = name.partition(".")
imports.add(cleaned_name)
packages = imports - (set(candidates) & imports)
logging.debug('Found packages: {0}'.format(packages))
logging.debug("Found packages: {0}".format(packages))
with open(join("stdlib"), "r") as f:
data = {x.strip() for x in f}
@ -157,42 +159,59 @@ def filter_line(line):
def generate_requirements_file(path, imports, symbol):
with _open(path, "w") as out_file:
logging.debug('Writing {num} requirements: {imports} to {file}'.format(
num=len(imports),
file=path,
imports=", ".join([x['name'] for x in imports])
))
fmt = '{name}' + symbol + '{version}'
out_file.write('\n'.join(
fmt.format(**item) if item['version'] else '{name}'.format(**item)
for item in imports) + '\n')
logging.debug(
"Writing {num} requirements: {imports} to {file}".format(
num=len(imports),
file=path,
imports=", ".join([x["name"] for x in imports]),
)
)
fmt = "{name}" + symbol + "{version}"
out_file.write(
"\n".join(
fmt.format(**item)
if item["version"]
else "{name}".format(**item)
for item in imports
)
+ "\n"
)
def output_requirements(imports, symbol):
generate_requirements_file('-', imports, symbol)
generate_requirements_file("-", imports, symbol)
def get_imports_info(
imports, pypi_server="https://pypi.python.org/pypi/", proxy=None, verify=None):
imports,
pypi_server="https://pypi.python.org/pypi/",
proxy=None,
verify=None,
):
result = []
for item in imports:
try:
response = requests.get(
"{0}{1}/json".format(pypi_server, item), proxies=proxy, verify=verify)
"{0}{1}/json".format(pypi_server, item),
proxies=proxy,
verify=verify,
)
if response.status_code == 200:
if hasattr(response.content, 'decode'):
if hasattr(response.content, "decode"):
data = json2package(response.content.decode())
else:
data = json2package(response.content)
elif response.status_code >= 300:
raise HTTPError(status_code=response.status_code,
reason=response.reason)
raise HTTPError(
status_code=response.status_code, reason=response.reason
)
except HTTPError:
logging.debug(
'Package %s does not exist or network problems', item)
"Package %s does not exist or network problems", item
)
continue
result.append({'name': item, 'version': data.latest_release_id})
result.append({"name": item, "version": data.latest_release_id})
return result
@ -212,16 +231,20 @@ def get_locally_installed_packages(encoding=None):
# TODO: What errors do we intend to suppress here?
continue
for i_item in package_import:
if ((i_item not in ignore) and
(package[0] not in ignore)):
if (i_item not in ignore) and (
package[0] not in ignore
):
version = None
if len(package) > 1:
version = package[1].replace(
".dist", "").replace(".egg", "")
version = (
package[1]
.replace(".dist", "")
.replace(".egg", "")
)
packages[i_item] = {
'version': version,
'name': package[0]
"version": version,
"name": package[0],
}
return packages
@ -234,12 +257,7 @@ def get_import_local(imports, encoding=None):
result.append(local[item.lower()])
# removing duplicates of package/version
result_unique = [
dict(t)
for t in set([
tuple(d.items()) for d in result
])
]
result_unique = [dict(t) for t in set([tuple(d.items()) for d in result])]
return result_unique
@ -270,7 +288,7 @@ def get_name_without_alias(name):
match = REGEXP[0].match(name.strip())
if match:
name = match.groups(0)[0]
return name.partition(' as ')[0].partition('.')[0].strip()
return name.partition(" as ")[0].partition(".")[0].strip()
def join(f):
@ -356,7 +374,8 @@ def diff(file_, imports):
logging.info(
"The following modules are in {} but do not seem to be imported: "
"{}".format(file_, ", ".join(x for x in modules_not_imported)))
"{}".format(file_, ", ".join(x for x in modules_not_imported))
)
def clean(file_, imports):
@ -404,20 +423,22 @@ def dynamic_versioning(scheme, imports):
def init(args):
encoding = args.get('--encoding')
extra_ignore_dirs = args.get('--ignore')
follow_links = not args.get('--no-follow-links')
input_path = args['<path>']
encoding = args.get("--encoding")
extra_ignore_dirs = args.get("--ignore")
follow_links = not args.get("--no-follow-links")
input_path = args["<path>"]
if input_path is None:
input_path = os.path.abspath(os.curdir)
if extra_ignore_dirs:
extra_ignore_dirs = extra_ignore_dirs.split(',')
extra_ignore_dirs = extra_ignore_dirs.split(",")
candidates = get_all_imports(input_path,
encoding=encoding,
extra_ignore_dirs=extra_ignore_dirs,
follow_links=follow_links)
candidates = get_all_imports(
input_path,
encoding=encoding,
extra_ignore_dirs=extra_ignore_dirs,
follow_links=follow_links,
)
candidates = get_pkg_names(candidates)
logging.debug("Found imports: " + ", ".join(candidates))
pypi_server = "https://pypi.python.org/pypi/"
@ -427,30 +448,36 @@ def init(args):
pypi_server = args["--pypi-server"]
if args["--proxy"]:
proxy = {'http': args["--proxy"], 'https': args["--proxy"]}
proxy = {"http": args["--proxy"], "https": args["--proxy"]}
if args["--verify"]:
verify = args["--verify"]
if args["--use-local"]:
logging.debug(
"Getting package information ONLY from local installation.")
"Getting package information ONLY from local installation."
)
imports = get_import_local(candidates, encoding=encoding)
else:
logging.debug("Getting packages information from Local/PyPI")
local = get_import_local(candidates, encoding=encoding)
# Get packages that were not found locally
difference = [x for x in candidates
if x.lower() not in [z['name'].lower() for z in local]]
imports = local + get_imports_info(difference,
proxy=proxy,
verify=verify,
pypi_server=pypi_server)
difference = [
x
for x in candidates
if x.lower() not in [z["name"].lower() for z in local]
]
imports = local + get_imports_info(
difference, proxy=proxy, verify=verify, pypi_server=pypi_server
)
# sort imports based on lowercase name of package, similar to `pip freeze`.
imports = sorted(imports, key=lambda x: x['name'].lower())
imports = sorted(imports, key=lambda x: x["name"].lower())
path = (args["--savepath"] if args["--savepath"] else
os.path.join(input_path, "requirements.txt"))
path = (
args["--savepath"]
if args["--savepath"]
else os.path.join(input_path, "requirements.txt")
)
if args["--diff"]:
diff(args["--diff"], imports)
@ -460,12 +487,15 @@ def init(args):
clean(args["--clean"], imports)
return
if (not args["--print"]
and not args["--savepath"]
and not args["--force"]
and os.path.exists(path)):
logging.warning("requirements.txt already exists, "
"use --force to overwrite it")
if (
not args["--print"]
and not args["--savepath"]
and not args["--force"]
and os.path.exists(path)
):
logging.warning(
"requirements.txt already exists, " "use --force to overwrite it"
)
return
if args["--mode"]:
@ -473,8 +503,10 @@ def init(args):
if scheme in ["compat", "gt", "no-pin"]:
imports, symbol = dynamic_versioning(scheme, imports)
else:
raise ValueError("Invalid argument for mode flag, "
"use 'compat', 'gt' or 'no-pin' instead")
raise ValueError(
"Invalid argument for mode flag, "
"use 'compat', 'gt' or 'no-pin' instead"
)
else:
symbol = "=="
@ -488,8 +520,8 @@ def init(args):
def main(): # pragma: no cover
args = docopt(__doc__, version=__version__)
log_level = logging.DEBUG if args['--debug'] else logging.INFO
logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s')
log_level = logging.DEBUG if args["--debug"] else logging.INFO
logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
try:
init(args)
@ -497,5 +529,5 @@ def main(): # pragma: no cover
sys.exit(0)
if __name__ == '__main__':
if __name__ == "__main__":
main() # pragma: no cover

View File

@ -1,56 +1,51 @@
"""unused import"""
# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
from __future__ import print_function
import atexit
import curses
import importlib # html/notebookapp.py
import logging
import os
import os.path as test # [unused-import]
import signal
import sqlite3
import sys
import time
import xml.etree # [unused-import]
import xml.sax # [unused-import]
import os.path as test # [unused-import]
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# +1:[unused-import,unused-import]
from collections import deque, OrderedDict, Counter
# All imports above should be ignored
import requests # [unused-import]
# astroid
# setuptools
import zipimport # command/easy_install.py
import zipimport # manager.py
# +1:[unused-import,unused-import]
from collections import Counter, OrderedDict, deque
# twisted
from importlib import invalidate_caches # python/test/test_deprecate.py
# astroid
import zipimport # manager.py
# IPython
from importlib.machinery import all_suffixes # core/completerlib.py
import importlib # html/notebookapp.py
from IPython.utils.importstring import import_item # Many files
# pyflakes
# test/test_doctests.py
from pyflakes.test.test_imports import Test as TestImports
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# see issue #88
import analytics
import flask_seasurf
import atexit
from __future__ import print_function
from docopt import docopt
import curses, logging, sqlite3
import logging
import os
import sqlite3
import time
import sys
import signal
import boto as b
import bs4
import nonexistendmodule
import boto as b, peewee as p
# import django
import flask.ext.somext # # #
import flask_seasurf
import nonexistendmodule
import peewee as p
# All imports above should be ignored
import requests # [unused-import]
from docopt import docopt
from IPython.utils.importstring import import_item # Many files
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# pyflakes
# test/test_doctests.py
from pyflakes.test.test_imports import Test as TestImports
from sqlalchemy import model
try:
import ujson as json
except ImportError:
@ -62,4 +57,5 @@ import models
def main():
pass
import after_method_is_valid_even_if_not_pep8

View File

@ -1,55 +1,50 @@
"""unused import"""
# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
from __future__ import print_function
import atexit
import curses
import importlib # html/notebookapp.py
import logging
import os
import os.path as test # [unused-import]
import signal
import sqlite3
import sys
import time
import xml.etree # [unused-import]
import xml.sax # [unused-import]
import os.path as test # [unused-import]
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# +1:[unused-import,unused-import]
from collections import deque, OrderedDict, Counter
# All imports above should be ignored
import requests # [unused-import]
# astroid
# setuptools
import zipimport # command/easy_install.py
import zipimport # manager.py
# +1:[unused-import,unused-import]
from collections import Counter, OrderedDict, deque
# twisted
from importlib import invalidate_caches # python/test/test_deprecate.py
# astroid
import zipimport # manager.py
# IPython
from importlib.machinery import all_suffixes # core/completerlib.py
import importlib # html/notebookapp.py
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# see issue #88
import analytics
import boto as b
import bs4
# import django
import flask.ext.somext # # #
import flask_seasurf
import nonexistendmodule
import peewee as p
# All imports above should be ignored
import requests # [unused-import]
from docopt import docopt
from IPython.utils.importstring import import_item # Many files
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# pyflakes
# test/test_doctests.py
from pyflakes.test.test_imports import Test as TestImports
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# see issue #88
import analytics
import flask_seasurf
import atexit
from __future__ import print_function
from docopt import docopt
import curses, logging, sqlite3
import logging
import os
import sqlite3
import time
import sys
import signal
import bs4
import nonexistendmodule
import boto as b, peewee as p
# import django
import flask.ext.somext # # #
# from sqlalchemy import model
try:
import ujson as json
@ -62,4 +57,5 @@ import models
def main():
pass
import after_method_is_valid_even_if_not_pep8

View File

@ -1,2 +1,2 @@
# Everything in here should be ignored
from pattern.web import Twitter, plaintext
from pattern.web import Twitter, plaintext

View File

@ -1,52 +1,48 @@
"""unused import"""
# pylint: disable=undefined-all-variable, import-error, no-absolute-import, too-few-public-methods, missing-docstring
from __future__ import print_function
import atexit
import curses
import importlib # html/notebookapp.py
import logging
import os
import os.path as test # [unused-import]
import signal
import sqlite3
import sys
import time
import xml.etree # [unused-import]
import xml.sax # [unused-import]
import os.path as test # [unused-import]
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
# +1:[unused-import,unused-import]
from collections import deque, OrderedDict, Counter
# All imports above should be ignored
import requests # [unused-import]
# astroid
# setuptools
import zipimport # command/easy_install.py
import zipimport # manager.py
# +1:[unused-import,unused-import]
from collections import Counter, OrderedDict, deque
# twisted
from importlib import invalidate_caches # python/test/test_deprecate.py
# astroid
import zipimport # manager.py
# IPython
from importlib.machinery import all_suffixes # core/completerlib.py
import importlib # html/notebookapp.py
from sys import argv as test2 # [unused-import]
from sys import flags # [unused-import]
import boto as b
import bs4
# import django
import flask.ext.somext # # #
import nonexistendmodule
import peewee as p
# All imports above should be ignored
import requests # [unused-import]
from docopt import docopt
from IPython.utils.importstring import import_item # Many files
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
# pyflakes
# test/test_doctests.py
from pyflakes.test.test_imports import Test as TestImports
# Nose
from nose.importer import Importer, add_path, remove_path # loader.py
import atexit
from __future__ import print_function
from docopt import docopt
import curses, logging, sqlite3
import logging
import os
import sqlite3
import time
import sys
import signal
import bs4
import nonexistendmodule
import boto as b, peewee as p
# import django
import flask.ext.somext # # #
from sqlalchemy import model
try:
import ujson as json
except ImportError:
@ -58,4 +54,5 @@ import models
def main():
pass
import after_method_is_valid_even_if_not_pep8

View File

@ -8,8 +8,9 @@ test_pipreqs
Tests for `pipreqs` module.
"""
import unittest
import os
import unittest
import requests
from pipreqs import pipreqs
@ -145,7 +146,7 @@ class TestPipreqs(unittest.TestCase):
Test that we can save requirements.txt correctly
to a different path
"""
pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path,
pipreqs.init({'<path>': self.project, '--savepath': self.alt_requirement_path,
'--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.alt_requirement_path) == 1
@ -163,7 +164,7 @@ class TestPipreqs(unittest.TestCase):
"""
with open(self.requirements_path, "w") as f:
f.write("should_not_be_overwritten")
pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None,
pipreqs.init({'<path>': self.project, '--savepath': None, '--use-local': None,
'--force': None, '--proxy':None, '--pypi-server':None, '--print': False,
'--diff': None, '--clean': None, '--mode': None})
assert os.path.exists(self.requirements_path) == 1
@ -203,7 +204,7 @@ class TestPipreqs(unittest.TestCase):
Test --ignore parameter
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy':None, '--pypi-server':None,
'--ignore':'.ignored_dir,.ignore_second',
@ -222,7 +223,7 @@ class TestPipreqs(unittest.TestCase):
Test --mode=no-pin
"""
pipreqs.init(
{'<path>': self.project_with_ignore_directory, '--savepath': None,
{'<path>': self.project_with_ignore_directory, '--savepath': None,
'--print': False, '--use-local': None, '--force': True,
'--proxy': None, '--pypi-server': None,
'--diff': None,