Merge branch 'master' into master

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                python-version: [3.6, 3.7, 3.8, 3.9, pypy-3.7]
+                python-version: [3.7, 3.8, 3.9, pypy-3.7]
 
         steps:
             - name: Checkout repository

CONTRIBUTING.rst

@@ -99,7 +99,7 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the list in README.rst.
-3. The pull request should work for Python 2.7, 3.4, 3.5, 3.6, and PyPy. Check
+3. The pull request should work for Python 3.7 to 3.11, and PyPy. Check
    https://travis-ci.org/bndr/pipreqs/pull_requests and make sure that the
    tests pass for all supported Python versions.
 

pipreqs/__init__.py

@@ -1,3 +1,3 @@
 __author__ = 'Vadim Kravcenko'
 __email__ = 'vadim.kravcenko@gmail.com'
-__version__ = '0.4.11'
+__version__ = '0.4.13'

pipreqs/mapping

@@ -10,6 +10,7 @@ BeautifulSoupTests:BeautifulSoup
 BioSQL:biopython
 BuildbotStatusShields:BuildbotEightStatusShields
 ComputedAttribute:ExtensionClass
+constraint:python-constraint
 Crypto:pycryptodome
 Cryptodome:pycryptodomex
 FSM:pexpect
@@ -129,6 +130,7 @@ aios3:aio_s3
 airbrake:airbrake_flask
 airship:airship_icloud
 airship:airship_steamcloud
+airflow:apache-airflow
 akamai:edgegrid_python
 alation:alation_api
 alba_client:alba_client_python
@@ -1030,9 +1032,10 @@ skbio:scikit_bio
 sklearn:scikit_learn
 slack:slackclient
 slugify:unicode_slugify
+slugify:python-slugify
 smarkets:smk_python_sdk
 snappy:ctypes_snappy
-socketio:gevent_socketio
+socketio:python-socketio
 socketserver:pies2overrides
 sockjs:sockjs_tornado
 socks:SocksiPy_branch
@@ -1061,6 +1064,7 @@ tasksitter:cerebrod
 tastypie:django_tastypie
 teamcity:teamcity_messages
 telebot:pyTelegramBotAPI
+telegram:python-telegram-bot
 tempita:Tempita
 tenjin:Tenjin
 termstyle:python_termstyle

pipreqs/pipreqs.py

@@ -176,6 +176,11 @@ def get_imports_info(
 
     for item in imports:
         try:
+            logging.warning(
+                'Import named "%s" not found locally. '
+                'Trying to resolve it at the PyPI server.',
+                item
+            )
             response = requests.get(
                 "{0}{1}/json".format(pypi_server, item), proxies=proxy)
             if response.status_code == 200:
@@ -187,15 +192,24 @@ def get_imports_info(
                 raise HTTPError(status_code=response.status_code,
                                 reason=response.reason)
         except HTTPError:
-            logging.debug(
+            logging.warning(
-                'Package %s does not exist or network problems', item)
+                'Package "%s" does not exist or network problems', item)
             continue
+        logging.warning(
+            'Import named "%s" was resolved to "%s:%s" package (%s).\n'
+            'Please, verify manually the final list of requirements.txt '
+            'to avoid possible dependency confusions.',
+            item,
+            data.name,
+            data.latest_release_id,
+            data.pypi_url
+        )
         result.append({'name': item, 'version': data.latest_release_id})
     return result
 
 
 def get_locally_installed_packages(encoding=None):
-    packages = {}
+    packages = []
     ignore = ["tests", "_tests", "egg", "EGG", "info"]
     for path in sys.path:
         for root, dirs, files in os.walk(path):
@@ -205,22 +219,36 @@ def get_locally_installed_packages(encoding=None):
                     with open(item, "r", encoding=encoding) as f:
                         package = root.split(os.sep)[-1].split("-")
                         try:
-                            package_import = f.read().strip().split("\n")
+                            top_level_modules = f.read().strip().split("\n")
                         except Exception:  # NOQA
+
                             # TODO: What errors do we intend to suppress here?
                             continue
-                        for i_item in package_import:
-                            if ((i_item not in ignore) and
-                                    (package[0] not in ignore)):
-                                version = None
-                                if len(package) > 1:
-                                    version = package[1].replace(
-                                        ".dist", "").replace(".egg", "")
 
-                                packages[i_item] = {
+                        # filter off explicitly ignored top-level modules
-                                    'version': version,
+                        # such as test, egg, etc.
-                                    'name': package[0]
+                        filtered_top_level_modules = list()
-                                }
+
+                        for module in top_level_modules:
+                            if (
+                                (module not in ignore) and
+                                (package[0] not in ignore)
+                            ):
+                                # append exported top level modules to the list
+                                filtered_top_level_modules.append(module)
+
+                        version = None
+                        if len(package) > 1:
+                            version = package[1].replace(
+                                ".dist", "").replace(".egg", "")
+
+                        # append package: top_level_modules pairs
+                        # instead of top_level_module: package pairs
+                        packages.append({
+                            'name': package[0],
+                            'version': version,
+                            'exports': filtered_top_level_modules
+                        })
     return packages
 
 
@@ -228,16 +256,19 @@ def get_import_local(imports, encoding=None):
     local = get_locally_installed_packages()
     result = []
     for item in imports:
-        if item.lower() in local:
+        # search through local packages
-            result.append(local[item.lower()])
+        for package in local:
+            # if candidate import name matches export name
+            # or candidate import name equals to the package name
+            # append it to the result
+            if item in package['exports'] or item == package['name']:
+                result.append(package)
 
     # removing duplicates of package/version
-    result_unique = [
+    # had to use second method instead of the previous one,
-        dict(t)
+    # because we have a list in the 'exports' field
-        for t in set([
+    # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python
-            tuple(d.items()) for d in result
+    result_unique = [i for n, i in enumerate(result) if i not in result[n+1:]]
-        ])
-    ]
 
     return result_unique
 
@@ -412,6 +443,16 @@ def init(args):
     if extra_ignore_dirs:
         extra_ignore_dirs = extra_ignore_dirs.split(',')
 
+    path = (args["--savepath"] if args["--savepath"] else
+            os.path.join(input_path, "requirements.txt"))
+    if (not args["--print"]
+            and not args["--savepath"]
+            and not args["--force"]
+            and os.path.exists(path)):
+        logging.warning("requirements.txt already exists, "
+                        "use --force to overwrite it")
+        return
+
     candidates = get_all_imports(input_path,
                                  encoding=encoding,
                                  extra_ignore_dirs=extra_ignore_dirs,
@@ -433,18 +474,26 @@ def init(args):
     else:
         logging.debug("Getting packages information from Local/PyPI")
         local = get_import_local(candidates, encoding=encoding)
-        # Get packages that were not found locally
+
-        difference = [x for x in candidates
+        # check if candidate name is found in
-                      if x.lower() not in [z['name'].lower() for z in local]]
+        # the list of exported modules, installed locally
+        # and the package name is not in the list of local module names
+        # it add to difference
+        difference = [x for x in candidates if
+                      # aggregate all export lists into one
+                      # flatten the list
+                      # check if candidate is in exports
+                      x.lower() not in [y for x in local for y in x['exports']]
+                      and
+                      # check if candidate is package names
+                      x.lower() not in [x['name'] for x in local]]
+
         imports = local + get_imports_info(difference,
                                            proxy=proxy,
                                            pypi_server=pypi_server)
     # sort imports based on lowercase name of package, similar to `pip freeze`.
     imports = sorted(imports, key=lambda x: x['name'].lower())
 
-    path = (args["--savepath"] if args["--savepath"] else
-            os.path.join(input_path, "requirements.txt"))
-
     if args["--diff"]:
         diff(args["--diff"], imports)
         return
@@ -453,14 +502,6 @@ def init(args):
         clean(args["--clean"], imports)
         return
 
-    if (not args["--print"]
-            and not args["--savepath"]
-            and not args["--force"]
-            and os.path.exists(path)):
-        logging.warning("requirements.txt already exists, "
-                        "use --force to overwrite it")
-        return
-
     if args["--mode"]:
         scheme = args.get("--mode")
         if scheme in ["compat", "gt", "no-pin"]:

requirements.txt

@@ -1,3 +1,3 @@
-wheel==0.23.0
+wheel==0.38.1
 Yarg==0.1.9
 docopt==0.6.2

setup.py

@@ -43,10 +43,11 @@ setup(
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
     ],
     test_suite='tests',
     entry_points={
@@ -54,4 +55,5 @@ setup(
             'pipreqs=pipreqs.pipreqs:main',
         ],
     },
+    python_requires='>=3.7',
 )

tox.ini

@@ -1,9 +1,8 @@
 [tox]
-envlist = py36, py37, py38, py39, pypy3, flake8
+envlist = py37, py38, py39, pypy3, flake8
 
 [gh-actions]
 python =
-    3.6: py36
     3.7: py37
     3.8: py38
     3.9: py39