From 270f0e993387c4989fea087e247743c66dfd998f Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 18:00:30 +0200 Subject: [PATCH 01/19] Add function `parse_requirements` --- pipreqs/pipreqs.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 209d415..aa72e9c 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -222,6 +222,32 @@ def get_name_without_alias(name): def join(f): return os.path.join(os.path.dirname(__file__), f) +def parse_requirements(): + modules = [] + delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar + + with open("requirements.txt", "r") as f: + data = [x.strip() for x in f.readlines() if x != "\n"] + + parameters = [x for x in data if x.startswith("-")] + data = [x for x in data if x[0].isalpha()] + + for x in data: + if not any([y in x for y in delim]): # Check for modules w/o a specifier. + modules.append(x) + for y in x: + if y in delim: + module = x.split(y) + module_name = module[0] + module_version = module[::-1][0].replace("=", "") + module = {module_name: module_version} + + if module not in modules: + modules.append(module) + + break + + return modules + parameters def init(args): encoding = args.get('--encoding') From 0886898d8d39f219b5c19c956e778e4f60fa1d85 Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 18:04:02 +0200 Subject: [PATCH 02/19] Fix dict key:value in function `parse_requirements()` --- pipreqs/pipreqs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index aa72e9c..4c3042e 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -234,13 +234,13 @@ def parse_requirements(): for x in data: if not any([y in x for y in delim]): # Check for modules w/o a specifier. - modules.append(x) + modules.append({"name": x, "version": None}) for y in x: if y in delim: module = x.split(y) module_name = module[0] module_version = module[::-1][0].replace("=", "") - module = {module_name: module_version} + module = {"name": module_name, "version": module_version} if module not in modules: modules.append(module) From 3d87f2139258bf555e230a6bb04392f926e6ab71 Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 19:59:59 +0200 Subject: [PATCH 03/19] Add `file_` parameter to function `parse_requirements`, implement a try-except block in function `parse_requirements` --- pipreqs/pipreqs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 4c3042e..9e9c312 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -222,12 +222,16 @@ def get_name_without_alias(name): def join(f): return os.path.join(os.path.dirname(__file__), f) -def parse_requirements(): +def parse_requirements(file_): modules = [] delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar - with open("requirements.txt", "r") as f: - data = [x.strip() for x in f.readlines() if x != "\n"] + try: + with open(file_, "r") as f: + data = [x.strip() for x in f.readlines() if x != "\n"] + except OSError as e: + print(e) + sys.exit(0) parameters = [x for x in data if x.startswith("-")] data = [x for x in data if x[0].isalpha()] From cdb82c6d9bf1d0ffe0b6ef8fbd128412acb0af8a Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 20:13:51 +0200 Subject: [PATCH 04/19] Switch out print statement against `logging.error` --- pipreqs/pipreqs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 9e9c312..635f39e 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -230,7 +230,7 @@ def parse_requirements(file_): with open(file_, "r") as f: data = [x.strip() for x in f.readlines() if x != "\n"] except OSError as e: - print(e) + logging.error(e) sys.exit(0) parameters = [x for x in data if x.startswith("-")] From d463b92810b7bb3fa08f3626930dd8a1af58bbd2 Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 21:13:47 +0200 Subject: [PATCH 05/19] Return tuple consisting of modules,parameters instead of a concatenated list --- pipreqs/pipreqs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 635f39e..d8c5f52 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -251,7 +251,8 @@ def parse_requirements(file_): break - return modules + parameters + return (modules, parameters) + def init(args): encoding = args.get('--encoding') @@ -294,6 +295,7 @@ def init(args): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return + if args["--print"]: output_requirements(imports) logging.info("Successfully output requirements") From b6c53c6b021cb381e747d7363db6a6b73e968a92 Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 21:16:17 +0200 Subject: [PATCH 06/19] Begin function `clean` --- pipreqs/pipreqs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index d8c5f52..a7a6578 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -253,6 +253,8 @@ def parse_requirements(file_): return (modules, parameters) +def clean(file_, imports): + modules, parameters = parse_requirements(file_) def init(args): encoding = args.get('--encoding') From b048c55fe6a0b8bded13b58b3b1b2145c998931f Mon Sep 17 00:00:00 2001 From: kxrd Date: Fri, 2 Jun 2017 22:30:17 +0200 Subject: [PATCH 07/19] Add `clean` function --- pipreqs/pipreqs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index a7a6578..ea3601c 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -256,6 +256,13 @@ def parse_requirements(file_): def clean(file_, imports): modules, parameters = parse_requirements(file_) + imports = [imports[i]["name"] for i in range(len(imports))] + modules = [modules[i]["name"] for i in range(len(modules))] + modules_not_imported = set(modules) - set(imports) + + print("These modules are in {} but does not seem to be imported: " + "\n{}".format(file_, "\n".join(x for x in modules_not_imported))) + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') From 888537a898f39bcde8bdb62df1570815c3f8a812 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 20:01:42 +0200 Subject: [PATCH 08/19] Improve except block in function parse_requirements --- pipreqs/pipreqs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index ea3601c..3248522 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -229,9 +229,9 @@ def parse_requirements(file_): try: with open(file_, "r") as f: data = [x.strip() for x in f.readlines() if x != "\n"] - except OSError as e: - logging.error(e) - sys.exit(0) + except OSError: + logging.error("Failed on file {}".format(file_)) + raise parameters = [x for x in data if x.startswith("-")] data = [x for x in data if x[0].isalpha()] From 1055a7a28bfd8f9a628e92d57d2ac7dd7956c3f0 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 20:02:43 +0200 Subject: [PATCH 09/19] Add colon to error output for consistency --- pipreqs/pipreqs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 3248522..cccbe6b 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -230,7 +230,7 @@ def parse_requirements(file_): with open(file_, "r") as f: data = [x.strip() for x in f.readlines() if x != "\n"] except OSError: - logging.error("Failed on file {}".format(file_)) + logging.error("Failed on file: {}".format(file_)) raise parameters = [x for x in data if x.startswith("-")] From d8b497ea9128c0bdae2c7c4bc90d2091948b6339 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 20:43:54 +0200 Subject: [PATCH 10/19] Rename function clean to compare_modules, have it return a tuple of modules not imported, add docstring --- pipreqs/pipreqs.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index cccbe6b..fe1d310 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -253,15 +253,26 @@ def parse_requirements(file_): return (modules, parameters) -def clean(file_, imports): + +def compare_modules(file_, imports): + """Compare modules in a file to imported modules in a project. + + Args: + file_ (str): File to parse for modules to be compared. + imports (tuple): Modules being imported in the project. + + Returns: + tuple: The modules not imported in the project, but do exist in the + specified file. + """ modules, parameters = parse_requirements(file_) imports = [imports[i]["name"] for i in range(len(imports))] modules = [modules[i]["name"] for i in range(len(modules))] modules_not_imported = set(modules) - set(imports) - print("These modules are in {} but does not seem to be imported: " - "\n{}".format(file_, "\n".join(x for x in modules_not_imported))) + return modules_not_imported + def init(args): encoding = args.get('--encoding') From 882a0d3ec3cbc4a4b636d35f8ef7d92b836c9a83 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 20:52:17 +0200 Subject: [PATCH 11/19] Add docstring to function parse_requirements --- pipreqs/pipreqs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index fe1d310..01f8c7d 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -223,6 +223,17 @@ def join(f): return os.path.join(os.path.dirname(__file__), f) def parse_requirements(file_): + """Parse a requirements formatted file. + + Args: + file_: File to parse. + + Raises: + OSerror: If there's any issues accessing the file. + + Returns: + tuple: The contents of the file, excluding comments. + """ modules = [] delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar From a78203dc2b3fd1790c865a20177e618d037776e5 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 21:22:18 +0200 Subject: [PATCH 12/19] Improve function parse_requirements docstring --- pipreqs/pipreqs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 01f8c7d..431764a 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -225,6 +225,10 @@ def join(f): def parse_requirements(file_): """Parse a requirements formatted file. + Traverse a string until a delimiter is detected, then split at said + delimiter, get module name by element index, create a dict consisting of + module:version, and add dict to list of parsed modules. + Args: file_: File to parse. From 7549b1c416b30ec803eda0c61c9e3e3c4f07fbd0 Mon Sep 17 00:00:00 2001 From: kxrd Date: Sat, 10 Jun 2017 21:24:39 +0200 Subject: [PATCH 13/19] Add and implement function diff, improve inline comments in function parse_requirements --- pipreqs/pipreqs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 431764a..fb93beb 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -18,6 +18,7 @@ Options: --savepath Save the list of requirements in the given file --print Output the list of requirements in the standard output --force Overwrite existing requirements.txt + --diff Compare modules in requirements.txt to project imports. """ from __future__ import print_function, absolute_import import os @@ -239,7 +240,7 @@ def parse_requirements(file_): tuple: The contents of the file, excluding comments. """ modules = [] - delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar + delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar try: with open(file_, "r") as f: @@ -252,7 +253,7 @@ def parse_requirements(file_): data = [x for x in data if x[0].isalpha()] for x in data: - if not any([y in x for y in delim]): # Check for modules w/o a specifier. + if not any([y in x for y in delim]): # Check for modules w/o a specifier. modules.append({"name": x, "version": None}) for y in x: if y in delim: @@ -289,6 +290,14 @@ def compare_modules(file_, imports): return modules_not_imported +def diff(file_, imports): + """Display the difference between modules in file a and imported modules.""" + modules_not_imported = compare_modules(file_, imports) + + logging.info("The following modules are in {} but do not seem to be imported: " + "{}".format(file_, ", ".join(x for x in modules_not_imported))) + + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -326,6 +335,10 @@ def init(args): path = (args["--savepath"] if args["--savepath"] else os.path.join(args[''], "requirements.txt")) + if args["--diff"]: + diff(args["--diff"], imports) + return + if not args["--print"] and not args["--savepath"] and not args["--force"] and os.path.exists(path): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") From ac6ce860d0590ceb0d06b8d479bd849a462e4b7b Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 01:10:47 +0200 Subject: [PATCH 14/19] Improve function diff docstring, begin function clean --- pipreqs/pipreqs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index fb93beb..574b53c 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -291,12 +291,16 @@ def compare_modules(file_, imports): def diff(file_, imports): - """Display the difference between modules in file a and imported modules.""" + """Display the difference between modules in a file and imported modules.""" modules_not_imported = compare_modules(file_, imports) logging.info("The following modules are in {} but do not seem to be imported: " "{}".format(file_, ", ".join(x for x in modules_not_imported))) +def clean(file_, imports): + """Remove modules that aren't imported in project from file.""" + + def init(args): encoding = args.get('--encoding') From d3efb942d59d9c96cc11f8b146f7b48977034cca Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 21:55:12 +0200 Subject: [PATCH 15/19] Stop exctracting parameters in file, just return list of modules --- pipreqs/pipreqs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 574b53c..e848552 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -249,7 +249,6 @@ def parse_requirements(file_): logging.error("Failed on file: {}".format(file_)) raise - parameters = [x for x in data if x.startswith("-")] data = [x for x in data if x[0].isalpha()] for x in data: @@ -267,7 +266,7 @@ def parse_requirements(file_): break - return (modules, parameters) + return modules def compare_modules(file_, imports): @@ -281,7 +280,7 @@ def compare_modules(file_, imports): tuple: The modules not imported in the project, but do exist in the specified file. """ - modules, parameters = parse_requirements(file_) + modules = parse_requirements(file_) imports = [imports[i]["name"] for i in range(len(imports))] modules = [modules[i]["name"] for i in range(len(modules))] From 54be2d1c249a6c321883ee2a9c44263975b3c927 Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 22:03:21 +0200 Subject: [PATCH 16/19] Complete function clean --- pipreqs/pipreqs.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index e848552..909c9d1 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -19,6 +19,7 @@ Options: --print Output the list of requirements in the standard output --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. + --clean Clean up requirements.txt by removing modules that are not imported in project. """ from __future__ import print_function, absolute_import import os @@ -298,8 +299,22 @@ def diff(file_, imports): def clean(file_, imports): """Remove modules that aren't imported in project from file.""" + modules_not_imported = compare_modules(file_, imports) + re_remove = re.compile("|".join(modules_not_imported)) + to_write = [] + with open(file_, "r+") as f: + for i in f.readlines(): + if re_remove.match(i) is None: + to_write.append(i) + f.seek(0) + f.truncate() + + for i in to_write: + f.write(i) + + logging.info("Successfully cleaned up requirements in " + file_) def init(args): encoding = args.get('--encoding') @@ -342,6 +357,10 @@ def init(args): diff(args["--diff"], imports) return + if args["--clean"]: + clean(args["--clean"], imports) + return + if not args["--print"] and not args["--savepath"] and not args["--force"] and os.path.exists(path): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") From e7b8ddf72de550b701cf71bf7e4b2576163ee7be Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 22:22:27 +0200 Subject: [PATCH 17/19] Replace with statement with a try/except/else/finally block to narrow down problem in the future --- pipreqs/pipreqs.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 909c9d1..004fe05 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -244,11 +244,14 @@ def parse_requirements(file_): delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar try: - with open(file_, "r") as f: - data = [x.strip() for x in f.readlines() if x != "\n"] + f = open_func(file_, "r") except OSError: logging.error("Failed on file: {}".format(file_)) raise + else: + data = [x.strip() for x in f.readlines() if x != "\n"] + finally: + f.close() data = [x for x in data if x[0].isalpha()] @@ -303,16 +306,22 @@ def clean(file_, imports): re_remove = re.compile("|".join(modules_not_imported)) to_write = [] - with open(file_, "r+") as f: + try: + f = open_func(file_, "r+") + except OSError: + logging.error("Failed on file: {}".format(file_)) + raise + else: for i in f.readlines(): if re_remove.match(i) is None: to_write.append(i) - f.seek(0) f.truncate() for i in to_write: f.write(i) + finally: + f.close() logging.info("Successfully cleaned up requirements in " + file_) From 06e933ef3c913b90c924450e17936a0eb47fa03c Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 22:55:16 +0200 Subject: [PATCH 18/19] Add new options --- tests/test_pipreqs.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index 5179dfb..2b855fa 100755 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -86,7 +86,8 @@ class TestPipreqs(unittest.TestCase): Test that all modules we will test upon, are in requirements file """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None}) + '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -98,7 +99,8 @@ class TestPipreqs(unittest.TestCase): Test that items listed in requirements.text are the same as locals expected """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, - '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None}) + '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, + '--diff': None, '--clean': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -111,7 +113,8 @@ class TestPipreqs(unittest.TestCase): Test that we can save requiremnts.tt correctly to a different path """ pipreqs.init({'': self.project, '--savepath': - self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False}) + self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -127,7 +130,8 @@ class TestPipreqs(unittest.TestCase): with open(self.requirements_path, "w") as f: f.write("should_not_be_overwritten") pipreqs.init({'': self.project, '--savepath': None, - '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False}) + '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, + "--diff": None, "--clean": None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -161,8 +165,10 @@ class TestPipreqs(unittest.TestCase): '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--ignore':'.ignored_dir,.ignore_second' - } + '--ignore':'.ignored_dir,.ignore_second', + '--diff': None, + '--clean': None + } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() From 254e1cedcb34dc3c645d027420685a7a079e5f62 Mon Sep 17 00:00:00 2001 From: kxrd Date: Tue, 13 Jun 2017 23:10:14 +0200 Subject: [PATCH 19/19] Improve variable module_version --- pipreqs/pipreqs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 004fe05..a739549 100755 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -262,7 +262,7 @@ def parse_requirements(file_): if y in delim: module = x.split(y) module_name = module[0] - module_version = module[::-1][0].replace("=", "") + module_version = module[-1].replace("=", "") module = {"name": module_name, "version": module_version} if module not in modules: