From 3f5964fcb90ec6eb6df46d78e651a1b73538d0ba Mon Sep 17 00:00:00 2001 From: adeadfed Date: Tue, 14 Mar 2023 21:14:19 +0100 Subject: [PATCH 1/9] fix name resolution for local packages --- pipreqs/pipreqs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 0ceb402..4195a75 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -229,7 +229,8 @@ def get_import_local(imports, encoding=None): result = [] for item in imports: if item.lower() in local: - result.append(local[item.lower()]) + # append to result a matching package, as well as its exported modules + result.append(dict(**local[item.lower()], exports=item.lower())) # removing duplicates of package/version result_unique = [ @@ -443,9 +444,12 @@ def init(args): else: logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) + # Get packages that were not found locally - difference = [x for x in candidates - if x.lower() not in [z['name'].lower() for z in local]] + difference = [x for x in candidates + # check if candidate name is found in the list of exported modules, installed locally + if x.lower() not in [y['exports'] for y in local]] + imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) From 6cd9925a31adc75522f4c453169eba162654af70 Mon Sep 17 00:00:00 2001 From: adeadfed Date: Wed, 29 Mar 2023 00:17:49 +0200 Subject: [PATCH 2/9] improved version of local package resolving --- pipreqs/pipreqs.py | 77 ++++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 4195a75..67a3110 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -195,7 +195,7 @@ def get_imports_info( def get_locally_installed_packages(encoding=None): - packages = {} + packages = [] ignore = ["tests", "_tests", "egg", "EGG", "info"] for path in sys.path: for root, dirs, files in os.walk(path): @@ -205,22 +205,31 @@ def get_locally_installed_packages(encoding=None): with open(item, "r", encoding=encoding) as f: package = root.split(os.sep)[-1].split("-") try: - package_import = f.read().strip().split("\n") + top_level_modules = f.read().strip().split("\n") except: # NOQA # TODO: What errors do we intend to suppress here? continue - for i_item in package_import: - if ((i_item not in ignore) and - (package[0] not in ignore)): - version = None - if len(package) > 1: - version = package[1].replace( - ".dist", "").replace(".egg", "") + + # filter off explicitly ignored top-level modules, such as test, egg, etc. + filtered_top_level_modules = list() - packages[i_item] = { - 'version': version, - 'name': package[0] - } + for module in top_level_modules: + if ((module not in ignore) and + (package[0] not in ignore)): + # append valid exported top level modules to the final list + filtered_top_level_modules.append(module) + + version = None + if len(package) > 1: + version = package[1].replace( + ".dist", "").replace(".egg", "") + + # append package: top_level_modules pairs instead of top_level_module: package pairs + packages.append({ + 'name': package[0], + 'version': version, + 'exports': filtered_top_level_modules + }) return packages @@ -228,17 +237,24 @@ def get_import_local(imports, encoding=None): local = get_locally_installed_packages() result = [] for item in imports: - if item.lower() in local: - # append to result a matching package, as well as its exported modules - result.append(dict(**local[item.lower()], exports=item.lower())) + # search through local packages + for package in local: + # if candidate import name matches export name inside the package exports + # or candidate import name equals to the package name + # append it to the result + if item in package['exports'] or item == package['name']: + result.append(package) + + # sanity check: + # imo we should no longer need the functionality to remove the duplicates + # TODO: recheck this + # it is possible that the pipreqs will write two packages, using same exported name again (!) + # we should log about this in case this happens # removing duplicates of package/version - result_unique = [ - dict(t) - for t in set([ - tuple(d.items()) for d in result - ]) - ] + # had to use second method instead of the first, listed here, because we have a list in the 'exports' field + # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python + result_unique = [i for n, i in enumerate(result) if i not in result[n + 1:]] return result_unique @@ -445,11 +461,18 @@ def init(args): logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) - # Get packages that were not found locally - difference = [x for x in candidates - # check if candidate name is found in the list of exported modules, installed locally - if x.lower() not in [y['exports'] for y in local]] - + # check if candidate name is found in the list of exported modules, installed locally + # and the package name is not in the list of local module names + # it add to difference + difference = [x for x in candidates + if + # aggregate all export lists into one + # flatten the list + x.lower() not in [y for x in local for y in x['exports']] + and + x.lower() not in [x['name'] for x in local] + ] + imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) From da442e7a2a3664c6001144a68bfcb63d72d7d6b3 Mon Sep 17 00:00:00 2001 From: adeadfed Date: Wed, 29 Mar 2023 00:25:45 +0200 Subject: [PATCH 3/9] add warnings to for remote resolving mode --- .vscode/launch.json | 18 ++++++++++++++++++ pipreqs/pipreqs.py | 11 +++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4dcfe00 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}\\pipreqs", + "justMyCode": true, + "args": ["C:/Users/Strawberry/Desktop/projects/pipreqs-vulnerable", "--print"] + } + ] +} \ No newline at end of file diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 67a3110..63e76a9 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -176,6 +176,7 @@ def get_imports_info( for item in imports: try: + logging.warning('Import "%s" not found locally. Trying to resolve it at the PyPI server.', item) response = requests.get( "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: @@ -187,9 +188,15 @@ def get_imports_info( raise HTTPError(status_code=response.status_code, reason=response.reason) except HTTPError: - logging.debug( - 'Package %s does not exist or network problems', item) + logging.warning( + 'Package "%s" does not exist or network problems', item) continue + logging.warning('Import "%s" was resolved to "%s:%s" package (%s).\nPlease, verify manually the final list of requirements.txt to avoid possible dependency confusions.', + item, + data.name, + data.latest_release_id, + data.pypi_url + ) result.append({'name': item, 'version': data.latest_release_id}) return result From 50498d3cd4b6ffbe30e93894e44e60b73b63d2f0 Mon Sep 17 00:00:00 2001 From: adeadfed Date: Wed, 29 Mar 2023 00:27:13 +0200 Subject: [PATCH 4/9] change warning messages --- pipreqs/pipreqs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 63e76a9..4690dcf 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -176,7 +176,7 @@ def get_imports_info( for item in imports: try: - logging.warning('Import "%s" not found locally. Trying to resolve it at the PyPI server.', item) + logging.warning('Import named "%s" not found locally. Trying to resolve it at the PyPI server.', item) response = requests.get( "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: @@ -191,7 +191,7 @@ def get_imports_info( logging.warning( 'Package "%s" does not exist or network problems', item) continue - logging.warning('Import "%s" was resolved to "%s:%s" package (%s).\nPlease, verify manually the final list of requirements.txt to avoid possible dependency confusions.', + logging.warning('Import named "%s" was resolved to "%s:%s" package (%s).\nPlease, verify manually the final list of requirements.txt to avoid possible dependency confusions.', item, data.name, data.latest_release_id, From 001ead91edea0837e9291176fce068f3e4f1211d Mon Sep 17 00:00:00 2001 From: adeadfed Date: Wed, 29 Mar 2023 00:28:17 +0200 Subject: [PATCH 5/9] delete vscode related files from repo --- .gitignore | 3 +++ .vscode/launch.json | 18 ------------------ 2 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index 45a067a..922004b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ Session.vim *~ /pipreqs/*.bak + +# vscode gitignore +.vscode/* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index 4dcfe00..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "cwd": "${workspaceFolder}\\pipreqs", - "justMyCode": true, - "args": ["C:/Users/Strawberry/Desktop/projects/pipreqs-vulnerable", "--print"] - } - ] -} \ No newline at end of file From 3ae82087f96ef1526ba1e0b93e0e7b7d72ddd669 Mon Sep 17 00:00:00 2001 From: adeadfed Date: Wed, 29 Mar 2023 00:32:00 +0200 Subject: [PATCH 6/9] remove invalid comment --- pipreqs/pipreqs.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 4690dcf..dc31e57 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -252,12 +252,6 @@ def get_import_local(imports, encoding=None): if item in package['exports'] or item == package['name']: result.append(package) - # sanity check: - # imo we should no longer need the functionality to remove the duplicates - # TODO: recheck this - # it is possible that the pipreqs will write two packages, using same exported name again (!) - # we should log about this in case this happens - # removing duplicates of package/version # had to use second method instead of the first, listed here, because we have a list in the 'exports' field # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python From bb61883079ebbe45714d3cb5cd035b951bc10885 Mon Sep 17 00:00:00 2001 From: adeadfed Date: Sat, 8 Apr 2023 17:10:02 +0200 Subject: [PATCH 7/9] revert changes to .gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 922004b..45a067a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,3 @@ Session.vim *~ /pipreqs/*.bak - -# vscode gitignore -.vscode/* \ No newline at end of file From 5537bcd217eff939c2e1835a1cac928ce429056f Mon Sep 17 00:00:00 2001 From: adeadfed Date: Thu, 13 Apr 2023 15:59:50 +0200 Subject: [PATCH 8/9] fix pep8 linting issues --- pipreqs/pipreqs.py | 69 +++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index dc31e57..75a6d0b 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -176,7 +176,11 @@ def get_imports_info( for item in imports: try: - logging.warning('Import named "%s" not found locally. Trying to resolve it at the PyPI server.', item) + logging.warning( + 'Import named "%s" not found locally.' + 'Trying to resolve it at the PyPI server.', + item + ) response = requests.get( "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: @@ -191,11 +195,14 @@ def get_imports_info( logging.warning( 'Package "%s" does not exist or network problems', item) continue - logging.warning('Import named "%s" was resolved to "%s:%s" package (%s).\nPlease, verify manually the final list of requirements.txt to avoid possible dependency confusions.', - item, - data.name, - data.latest_release_id, - data.pypi_url + logging.warning( + 'Import named "%s" was resolved to "%s:%s" package (%s).' + 'Please, verify manually the final list of requirements.txt' + 'to avoid possible dependency confusions.', + item, + data.name, + data.latest_release_id, + data.pypi_url ) result.append({'name': item, 'version': data.latest_release_id}) return result @@ -216,22 +223,26 @@ def get_locally_installed_packages(encoding=None): except: # NOQA # TODO: What errors do we intend to suppress here? continue - - # filter off explicitly ignored top-level modules, such as test, egg, etc. + + # filter off explicitly ignored top-level modules + # such as test, egg, etc. filtered_top_level_modules = list() for module in top_level_modules: - if ((module not in ignore) and - (package[0] not in ignore)): - # append valid exported top level modules to the final list - filtered_top_level_modules.append(module) - + if ( + (module not in ignore) and + (package[0] not in ignore) + ): + # append exported top level modules to the list + filtered_top_level_modules.append(module) + version = None if len(package) > 1: version = package[1].replace( ".dist", "").replace(".egg", "") - # append package: top_level_modules pairs instead of top_level_module: package pairs + # append package: top_level_modules pairs + # instead of top_level_module: package pairs packages.append({ 'name': package[0], 'version': version, @@ -246,16 +257,17 @@ def get_import_local(imports, encoding=None): for item in imports: # search through local packages for package in local: - # if candidate import name matches export name inside the package exports + # if candidate import name matches export name # or candidate import name equals to the package name # append it to the result if item in package['exports'] or item == package['name']: result.append(package) # removing duplicates of package/version - # had to use second method instead of the first, listed here, because we have a list in the 'exports' field + # had to use second method instead of the previous one, + # because we have a list in the 'exports' field # https://stackoverflow.com/questions/9427163/remove-duplicate-dict-in-list-in-python - result_unique = [i for n, i in enumerate(result) if i not in result[n + 1:]] + result_unique = [i for n, i in enumerate(result) if i not in result[n+1:]] return result_unique @@ -461,19 +473,20 @@ def init(args): else: logging.debug("Getting packages information from Local/PyPI") local = get_import_local(candidates, encoding=encoding) - - # check if candidate name is found in the list of exported modules, installed locally + + # check if candidate name is found in + # the list of exported modules, installed locally # and the package name is not in the list of local module names # it add to difference - difference = [x for x in candidates - if - # aggregate all export lists into one - # flatten the list - x.lower() not in [y for x in local for y in x['exports']] - and - x.lower() not in [x['name'] for x in local] - ] - + difference = [x for x in candidates if + # aggregate all export lists into one + # flatten the list + # check if candidate is in exports + x.lower() not in [y for x in local for y in x['exports']] + and + # check if candidate is package names + x.lower() not in [x['name'] for x in local]] + imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) From 2103371746b727f6db341d301214d16c0dfba59b Mon Sep 17 00:00:00 2001 From: adeadfed Date: Thu, 13 Apr 2023 16:05:50 +0200 Subject: [PATCH 9/9] add whitespaces to pep8 formatted strings --- pipreqs/pipreqs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 75a6d0b..8182115 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -177,7 +177,7 @@ def get_imports_info( for item in imports: try: logging.warning( - 'Import named "%s" not found locally.' + 'Import named "%s" not found locally. ' 'Trying to resolve it at the PyPI server.', item ) @@ -196,8 +196,8 @@ def get_imports_info( 'Package "%s" does not exist or network problems', item) continue logging.warning( - 'Import named "%s" was resolved to "%s:%s" package (%s).' - 'Please, verify manually the final list of requirements.txt' + 'Import named "%s" was resolved to "%s:%s" package (%s).\n' + 'Please, verify manually the final list of requirements.txt ' 'to avoid possible dependency confusions.', item, data.name,