diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 288065e..837fcfb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.7] + python-version: [3.7, 3.8, 3.9, pypy-3.7] steps: - name: Checkout repository diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index e6ae58a..60e73a2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -99,7 +99,7 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 2.7, 3.4, 3.5, 3.6, and PyPy. Check +3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the tests pass for all supported Python versions. diff --git a/pipreqs/__init__.py b/pipreqs/__init__.py index 0491276..d163f29 100755 --- a/pipreqs/__init__.py +++ b/pipreqs/__init__.py @@ -1,3 +1,3 @@ __author__ = 'Vadim Kravcenko' __email__ = 'vadim.kravcenko@gmail.com' -__version__ = '0.4.11' +__version__ = '0.4.13' diff --git a/pipreqs/mapping b/pipreqs/mapping index c1729eb..3e0dc37 100644 --- a/pipreqs/mapping +++ b/pipreqs/mapping @@ -10,6 +10,7 @@ BeautifulSoupTests:BeautifulSoup BioSQL:biopython BuildbotStatusShields:BuildbotEightStatusShields ComputedAttribute:ExtensionClass +constraint:python-constraint Crypto:pycryptodome Cryptodome:pycryptodomex FSM:pexpect @@ -129,6 +130,7 @@ aios3:aio_s3 airbrake:airbrake_flask airship:airship_icloud airship:airship_steamcloud +airflow:apache-airflow akamai:edgegrid_python alation:alation_api alba_client:alba_client_python @@ -1030,9 +1032,10 @@ skbio:scikit_bio sklearn:scikit_learn slack:slackclient slugify:unicode_slugify +slugify:python-slugify smarkets:smk_python_sdk snappy:ctypes_snappy -socketio:gevent_socketio +socketio:python-socketio socketserver:pies2overrides sockjs:sockjs_tornado socks:SocksiPy_branch @@ -1061,6 +1064,7 @@ tasksitter:cerebrod tastypie:django_tastypie teamcity:teamcity_messages telebot:pyTelegramBotAPI +telegram:python-telegram-bot tempita:Tempita tenjin:Tenjin termstyle:python_termstyle diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 001063d..51e57d8 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -176,6 +176,11 @@ def get_imports_info( for item in imports: try: + logging.warning( + 'Import named "%s" not found locally. ' + 'Trying to resolve it at the PyPI server.', + item + ) response = requests.get( "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: @@ -187,15 +192,24 @@ def get_imports_info( raise HTTPError(status_code=response.status_code, reason=response.reason) except HTTPError: - logging.debug( - 'Package %s does not exist or network problems', item) + logging.warning( + 'Package "%s" does not exist or network problems', item) continue + logging.warning( + 'Import named "%s" was resolved to "%s:%s" package (%s).\n' + 'Please, verify manually the final list of requirements.txt ' + 'to avoid possible dependency confusions.', + item, + data.name, + data.latest_release_id, + data.pypi_url + ) result.append({'name': item, 'version': data.latest_release_id}) return result def get_locally_installed_packages(encoding=None): - packages = {} + packages = [] ignore = ["tests", "_tests", "egg", "EGG", "info"] for path in sys.path: for root, dirs, files in os.walk(path): @@ -205,22 +219,36 @@ def get_locally_installed_packages(encoding=None): with open(item, "r", encoding=encoding) as f: package = root.split(os.sep)[-1].split("-") try: - package_import = f.read().strip().split("\n") + top_level_modules = f.read().strip().split("\n") except Exception: # NOQA + # TODO: What errors do we intend to suppress here? continue - for i_item in package_import: - if ((i_item not in ignore) and - (package[0] not in ignore)): - version = None - if len(package) > 1: - version = package[1].replace( - ".dist", "").replace(".egg", "") - packages[i_item] = { - 'version': version, - 'name': package[0] - } + # filter off explicitly ignored top-level modules + # such as test, egg, etc. + filtered_top_level_modules = list() + + for module in top_level_modules: + if ( + (module not in ignore) and + (package[0] not in ignore) + ): + # append exported top level modules to the list + filtered_top_level_modules.append(module) + + version = None + if len(package) > 1: + version = package[1].replace( + ".dist", "").replace(".egg", "") + + # append package: top_level_modules pairs + # instead of top_level_module: package pairs + packages.append({ + 'name': package[0], + 'version': version, + 'exports': filtered_top_level_modules + }) return packages @@ -228,16 +256,19 @@ def get_import_local(imports, encoding=None): local = get_locally_installed_packages() result = [] for item in imports: - if item.lower() in local: - result.append(local[item.lower()]) + # search through local packages + for package in local: + # if candidate import name matches export name + # or candidate import name equals to the package name + # append it to the result + if item in package['exports'] or item == package['name']: + result.append(package) # removing duplicates of package/version - result_unique = [ - dict(t) - for t in set([ - tuple(d.items()) for d in result - ]) - ] + # had to use second method instead of the previous one, + # because we have a list in the 'exports' field + # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python + result_unique = [i for n, i in enumerate(result) if i not in result[n+1:]] return result_unique @@ -412,6 +443,16 @@ def init(args): if extra_ignore_dirs: extra_ignore_dirs = extra_ignore_dirs.split(',') + path = (args["--savepath"] if args["--savepath"] else + os.path.join(input_path, "requirements.txt")) + if (not args["--print"] + and not args["--savepath"] + and not args["--force"] + and os.path.exists(path)): + logging.warning("requirements.txt already exists, " + "use --force to overwrite it") + return + candidates = get_all_imports(input_path, encoding=encoding, extra_ignore_dirs=extra_ignore_dirs, @@ -433,18 +474,26 @@ def init(args): else: logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) - # Get packages that were not found locally - difference = [x for x in candidates - if x.lower() not in [z['name'].lower() for z in local]] + + # check if candidate name is found in + # the list of exported modules, installed locally + # and the package name is not in the list of local module names + # it add to difference + difference = [x for x in candidates if + # aggregate all export lists into one + # flatten the list + # check if candidate is in exports + x.lower() not in [y for x in local for y in x['exports']] + and + # check if candidate is package names + x.lower() not in [x['name'] for x in local]] + imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) # sort imports based on lowercase name of package, similar to `pip freeze`. imports = sorted(imports, key=lambda x: x['name'].lower()) - path = (args["--savepath"] if args["--savepath"] else - os.path.join(input_path, "requirements.txt")) - if args["--diff"]: diff(args["--diff"], imports) return @@ -453,14 +502,6 @@ def init(args): clean(args["--clean"], imports) return - if (not args["--print"] - and not args["--savepath"] - and not args["--force"] - and os.path.exists(path)): - logging.warning("requirements.txt already exists, " - "use --force to overwrite it") - return - if args["--mode"]: scheme = args.get("--mode") if scheme in ["compat", "gt", "no-pin"]: diff --git a/requirements.txt b/requirements.txt index 959d1b7..f4c9e26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -wheel==0.23.0 +wheel==0.38.1 Yarg==0.1.9 docopt==0.6.2 \ No newline at end of file diff --git a/setup.py b/setup.py index f119905..8b826ed 100755 --- a/setup.py +++ b/setup.py @@ -43,10 +43,11 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], test_suite='tests', entry_points={ @@ -54,4 +55,5 @@ setup( 'pipreqs=pipreqs.pipreqs:main', ], }, + python_requires='>=3.7', ) diff --git a/tox.ini b/tox.ini index cf5c2c2..aea10ec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,8 @@ [tox] -envlist = py36, py37, py38, py39, pypy3, flake8 +envlist = py37, py38, py39, pypy3, flake8 [gh-actions] python = - 3.6: py36 3.7: py37 3.8: py38 3.9: py39