From 4eae4794a0548710099ae22ce5b9fcfd2a8845b7 Mon Sep 17 00:00:00 2001 From: Jake Teo Date: Thu, 25 Mar 2021 17:37:00 +0800 Subject: [PATCH 1/3] New Option for Dynamic Versioning (#1) * added new option for dynamic versioning * added quotes for dynamic options Co-authored-by: Siyang --- AUTHORS.rst | 3 ++- README.rst | 2 +- pipreqs/pipreqs.py | 34 +++++++++++++++++++++--- tests/test_pipreqs.py | 61 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 85 insertions(+), 15 deletions(-) mode change 100755 => 100644 pipreqs/pipreqs.py mode change 100755 => 100644 tests/test_pipreqs.py diff --git a/AUTHORS.rst b/AUTHORS.rst index eaa038a..2f31bac 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,4 +10,5 @@ Development Lead Contributors ------------ -None yet. Why not be the first? +* Jake Teo +* Jerome Chan diff --git a/README.rst b/README.rst index 139fa7d..93b1735 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ Usage --force Overwrite existing requirements.txt --diff Compare modules in requirements.txt to project imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --no-pin Omit version of output packages. + --dynamic Enables dynamic version updates by 'minor', 'micro' or 'all' schemes. Example ------- diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py old mode 100755 new mode 100644 index 57147e2..069c28a --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,7 +31,8 @@ Options: imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --no-pin Omit version of output packages. + --dynamic Enables dynamic version updates by 'minor', + 'micro' or 'all' schemes. """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -392,6 +393,28 @@ def clean(file_, imports): logging.info("Successfully cleaned up requirements in " + file_) +def dynamic_versioning(scheme, imports): + """Enables dynamic versioning by minor, micro or all scheme.""" + if scheme == "all": + imports = [{"name": item["name"], "version": ""} for item in imports] + else: + for item in imports: + version = item["version"] + arr = version.split(".") + length = len(arr) + if length != 1: + if scheme == "minor": + arr = arr[0] + + elif scheme == "micro" and length >= 2: + arr = arr[:2] + arr = ".".join(arr) + + arr = arr + ".*" + item["version"] = arr + return imports + + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') @@ -450,8 +473,13 @@ def init(args): "use --force to overwrite it") return - if args.get('--no-pin'): - imports = [{'name': item["name"], 'version': ''} for item in imports] + if args["--dynamic"]: + scheme = args.get("--dynamic") + if scheme in ["minor", "micro", "all"]: + imports = dynamic_versioning(scheme, imports) + else: + raise ValueError("Invalid argument for dynamic scheme, " + "use 'minor', 'micro' or 'all' instead") if args["--print"]: output_requirements(imports) diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py old mode 100755 new mode 100644 index dcd75c5..b150c9a --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -93,7 +93,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None}) + '--diff': None, '--clean': None, '--dynamic': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -106,7 +106,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None}) + '--diff': None, '--clean': None, '--dynamic': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -120,7 +120,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None}) + "--diff": None, "--clean": None, '--dynamic': None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -137,7 +137,7 @@ class TestPipreqs(unittest.TestCase): f.write("should_not_be_overwritten") pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None}) + "--diff": None, "--clean": None, '--dynamic': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -173,7 +173,8 @@ class TestPipreqs(unittest.TestCase): '--pypi-server':None, '--ignore':'.ignored_dir,.ignore_second', '--diff': None, - '--clean': None + '--clean': None, + '--dynamic': None } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -181,9 +182,9 @@ class TestPipreqs(unittest.TestCase): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) - def test_omit_version(self): + def test_dynamic_version_all_scheme(self): """ - Test --no-pin parameter + Test --dynamic=all """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -192,13 +193,53 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--no-pin': True + '--dynamic': 'all' } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.read().lower() - for item in ['beautifulsoup4==4.8.1', 'boto==2.49.0']: - self.assertFalse(item.lower() in data) + for item in ['beautifulsoup4', 'boto']: + self.assertTrue(item.lower() in data) + + def test_dynamic_version_micro_scheme(self): + """ + Test --dynamic=micro + """ + pipreqs.init( + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy': None, + '--pypi-server': None, + '--diff': None, + '--clean': None, + '--dynamic': 'micro' + } + ) + with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.readlines() + for item in data: + asterisk = item.strip().split(".")[2] + self.assertEqual(asterisk, "*",) + + def test_dynamic_version_minor_scheme(self): + """ + Test --dynamic=minor + """ + pipreqs.init( + {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, + '--use-local': None, '--force': True, + '--proxy': None, + '--pypi-server': None, + '--diff': None, + '--clean': None, + '--dynamic': 'minor' + } + ) + with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: + data = f.readlines() + for item in data: + asterisk = item.strip().split(".")[1] + self.assertEqual(asterisk, "*",) def tearDown(self): """ From 69a884a4c4980e0914d1de0056c2a3df496dd563 Mon Sep 17 00:00:00 2001 From: Siyang Date: Mon, 29 Mar 2021 21:35:34 +0800 Subject: [PATCH 2/3] changes based on discussions w maintainer --- pipreqs/pipreqs.py | 59 ++++++++++++++++++++----------------------- tests/test_pipreqs.py | 38 +++++++++++++++------------- 2 files changed, 48 insertions(+), 49 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 069c28a..05ce0be 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -31,8 +31,11 @@ Options: imports. --clean Clean up requirements.txt by removing modules that are not imported in project. - --dynamic Enables dynamic version updates by 'minor', - 'micro' or 'all' schemes. + --mode Enables dynamic versioning with , + or schemes. + | e.g. Flask~=1.1.2 + | e.g. Flask>=1.1.2 + | e.g. Flask """ from __future__ import print_function, absolute_import from contextlib import contextmanager @@ -162,21 +165,21 @@ def filter_line(line): return len(line) > 0 and line[0] != "#" -def generate_requirements_file(path, imports): +def generate_requirements_file(path, imports, symbol): with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, imports=", ".join([x['name'] for x in imports]) )) - fmt = '{name}=={version}' + fmt = '{name}' + symbol + '{version}' out_file.write('\n'.join( fmt.format(**item) if item['version'] else '{name}'.format(**item) for item in imports) + '\n') -def output_requirements(imports): - generate_requirements_file('-', imports) +def output_requirements(imports, symbol): + generate_requirements_file('-', imports, symbol) def get_imports_info( @@ -394,25 +397,17 @@ def clean(file_, imports): def dynamic_versioning(scheme, imports): - """Enables dynamic versioning by minor, micro or all scheme.""" - if scheme == "all": + """Enables dynamic versioning with , or schemes.""" + if scheme == "no-pin": imports = [{"name": item["name"], "version": ""} for item in imports] + symbol = "" + elif scheme == "gt": + symbol = ">=" + elif scheme == "compat": + symbol = "~=" else: - for item in imports: - version = item["version"] - arr = version.split(".") - length = len(arr) - if length != 1: - if scheme == "minor": - arr = arr[0] - - elif scheme == "micro" and length >= 2: - arr = arr[:2] - arr = ".".join(arr) - - arr = arr + ".*" - item["version"] = arr - return imports + symbol = "==" + return imports, symbol def init(args): @@ -473,19 +468,21 @@ def init(args): "use --force to overwrite it") return - if args["--dynamic"]: - scheme = args.get("--dynamic") - if scheme in ["minor", "micro", "all"]: - imports = dynamic_versioning(scheme, imports) + if args["--mode"]: + scheme = args.get("--mode") + if scheme in ["compat", "gt", "no-pin"]: + imports, symbol = dynamic_versioning(scheme, imports) else: - raise ValueError("Invalid argument for dynamic scheme, " - "use 'minor', 'micro' or 'all' instead") + raise ValueError("Invalid argument for mode flag, " + "use 'compat', 'gt' or 'no-pin' instead") + else: + symbol = "==" if args["--print"]: - output_requirements(imports) + output_requirements(imports, symbol) logging.info("Successfully output requirements") else: - generate_requirements_file(path, imports) + generate_requirements_file(path, imports, symbol) logging.info("Successfully saved requirements file in " + path) diff --git a/tests/test_pipreqs.py b/tests/test_pipreqs.py index b150c9a..1270058 100644 --- a/tests/test_pipreqs.py +++ b/tests/test_pipreqs.py @@ -93,7 +93,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': None, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--dynamic': None}) + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -106,7 +106,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': None, '--print': False, '--use-local': True, '--force': True, '--proxy':None, '--pypi-server':None, - '--diff': None, '--clean': None, '--dynamic': None}) + '--diff': None, '--clean': None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.readlines() @@ -120,7 +120,7 @@ class TestPipreqs(unittest.TestCase): """ pipreqs.init({'': self.project, '--savepath': self.alt_requirement_path, '--use-local': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None, '--dynamic': None}) + "--diff": None, "--clean": None, '--mode': None}) assert os.path.exists(self.alt_requirement_path) == 1 with open(self.alt_requirement_path, "r") as f: data = f.read().lower() @@ -137,7 +137,7 @@ class TestPipreqs(unittest.TestCase): f.write("should_not_be_overwritten") pipreqs.init({'': self.project, '--savepath': None, '--use-local': None, '--force': None, '--proxy':None, '--pypi-server':None, '--print': False, - "--diff": None, "--clean": None, '--dynamic': None}) + "--diff": None, "--clean": None, '--mode': None}) assert os.path.exists(self.requirements_path) == 1 with open(self.requirements_path, "r") as f: data = f.read().lower() @@ -174,7 +174,7 @@ class TestPipreqs(unittest.TestCase): '--ignore':'.ignored_dir,.ignore_second', '--diff': None, '--clean': None, - '--dynamic': None + '--mode': None } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -182,9 +182,9 @@ class TestPipreqs(unittest.TestCase): for item in ['click', 'getpass']: self.assertFalse(item.lower() in data) - def test_dynamic_version_all_scheme(self): + def test_dynamic_version_no_pin_scheme(self): """ - Test --dynamic=all + Test --mode=no-pin """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -193,7 +193,7 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--dynamic': 'all' + '--mode': 'no-pin' } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: @@ -201,9 +201,9 @@ class TestPipreqs(unittest.TestCase): for item in ['beautifulsoup4', 'boto']: self.assertTrue(item.lower() in data) - def test_dynamic_version_micro_scheme(self): + def test_dynamic_version_gt_scheme(self): """ - Test --dynamic=micro + Test --mode=gt """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -212,18 +212,19 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--dynamic': 'micro' + '--mode': 'gt' } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() for item in data: - asterisk = item.strip().split(".")[2] - self.assertEqual(asterisk, "*",) + symbol = '>=' + message = 'symbol is not in item' + self.assertIn(symbol, item, message) - def test_dynamic_version_minor_scheme(self): + def test_dynamic_version_compat_scheme(self): """ - Test --dynamic=minor + Test --mode=compat """ pipreqs.init( {'': self.project_with_ignore_directory, '--savepath': None, '--print': False, @@ -232,14 +233,15 @@ class TestPipreqs(unittest.TestCase): '--pypi-server': None, '--diff': None, '--clean': None, - '--dynamic': 'minor' + '--mode': 'compat' } ) with open(os.path.join(self.project_with_ignore_directory, "requirements.txt"), "r") as f: data = f.readlines() for item in data: - asterisk = item.strip().split(".")[1] - self.assertEqual(asterisk, "*",) + symbol = '~=' + message = 'symbol is not in item' + self.assertIn(symbol, item, message) def tearDown(self): """ From 5dc02aa2facbb0e9fdc4d9f42ed538adeefbae94 Mon Sep 17 00:00:00 2001 From: Jake Date: Mon, 29 Mar 2021 22:16:45 +0800 Subject: [PATCH 3/3] remove obsolete '==' --- pipreqs/pipreqs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index 04d9f47..20aae68 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -410,8 +410,6 @@ def dynamic_versioning(scheme, imports): symbol = ">=" elif scheme == "compat": symbol = "~=" - else: - symbol = "==" return imports, symbol