From 484a98b48f20638c5c334fd8238e8de94fed4cdc Mon Sep 17 00:00:00 2001 From: YYLIZH <48501277+YYLIZH@users.noreply.github.com> Date: Tue, 25 Apr 2023 06:24:22 +0800 Subject: [PATCH] feat: MakeFile easy to test local (#254) --- .github/workflows/make_test_ebook.yaml | 40 +--- Makefile | 10 + setup.py | 20 +- tests/test_integration.py | 312 +++++++++++++++++++++++++ 4 files changed, 336 insertions(+), 46 deletions(-) create mode 100644 Makefile create mode 100644 tests/test_integration.py diff --git a/.github/workflows/make_test_ebook.yaml b/.github/workflows/make_test_ebook.yaml index 38008f6..9bf4108 100644 --- a/.github/workflows/make_test_ebook.yaml +++ b/.github/workflows/make_test_ebook.yaml @@ -34,42 +34,8 @@ jobs: run: | pip install . - - name: make normal ebook test using google translate and cli + - name: Run tests run: | - bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 10 --model google --translate-tags div,p - bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 20 --model google - - - name: make txt book test using google translate - run: | - python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --test_num 20 --model google - - - name: make txt book test with batch_size - run: | - python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 20 --model google - - - name: make caiyun translator test - if: env.BBM_CAIYUN_API_KEY != null - run: | - python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 100 --model caiyun - - - name: make deepl translator test - if: env.BBM_CAIYUN_API_KEY != null - run: | - python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 20 --model deepl - python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test - - - name: make openai key ebook test - if: env.BBM_DEEPL_API_KEY != null - run: | - python3 make_book.py --book_name "test_books/lemo.epub" --test --test_num 5 --language zh-hans - python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --model gpt3 --prompt prompt_template_sample.txt - python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --prompt prompt_template_sample.json - python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test --test_num 20 - - - name: Rename and Upload ePub - if: env.OPENAI_API_KEY != null - uses: actions/upload-artifact@v2 - with: - name: epub_output - path: "test_books/lemo_bilingual.epub" + pip install pytest + pytest tests diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..674ff1e --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +SHELL := /bin/bash + +fmt: + @echo "Running formatter ..." + venv/bin/black . + +.PHONY:tests +tests: + @echo "Running tests ..." + venv/bin/pytest tests/test_integration.py \ No newline at end of file diff --git a/setup.py b/setup.py index 11de1e8..6412474 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,16 @@ #!/usr/bin/env python3 from setuptools import find_packages, setup + +def get_required_packges(): + packages = [] + with open("requirements.txt") as filep: + for line in filep: + packages.append(line.rstrip()) + + return packages + + setup( name="bbook_maker", description="The bilingual_book_maker is an AI translation tool that uses ChatGPT to assist users in creating multi-language versions of epub/txt files and books.", @@ -11,15 +21,7 @@ setup( packages=find_packages(), url="https://github.com/yihong0618/bilingual_book_maker", python_requires=">=3.7", - install_requires=[ - "bs4", - "openai", - "requests", - "ebooklib", - "rich", - "tqdm", - "tiktoken", - ], + install_requires=get_required_packges(), classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..8964df5 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,312 @@ +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture() +def test_book_dir() -> str: + """Return test book dir""" + # TODO: Can move this to conftest.py if there will be more unittests + return str(Path(__file__).parent.parent / "test_books") + + +def test_google_translate_epub(test_book_dir, tmpdir): + """Test google translate epub""" + shutil.copyfile( + os.path.join(test_book_dir, "Liber_Esther.epub"), + os.path.join(tmpdir, "Liber_Esther.epub"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "Liber_Esther.epub"), + "--test", + "--test_num", + "20", + "--model", + "google", + ], + env=os.environ.copy(), + ) + + assert os.path.isfile(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) + assert os.path.getsize(os.path.join(tmpdir, "Liber_Esther_bilingual.epub")) != 0 + + +def test_google_translate_epub_cli(): + pass + + +def test_google_translate_txt(test_book_dir, tmpdir): + """Test google translate txt""" + shutil.copyfile( + os.path.join(test_book_dir, "the_little_prince.txt"), + os.path.join(tmpdir, "the_little_prince.txt"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "the_little_prince.txt"), + "--test", + "--test_num", + "20", + "--model", + "google", + ], + env=os.environ.copy(), + ) + assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) + assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 + + +def test_google_translate_txt_batch_size(test_book_dir, tmpdir): + """Test google translate txt with batch_size""" + shutil.copyfile( + os.path.join(test_book_dir, "the_little_prince.txt"), + os.path.join(tmpdir, "the_little_prince.txt"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "the_little_prince.txt"), + "--test", + "--batch_size", + "30", + "--test_num", + "20", + "--model", + "google", + ], + env=os.environ.copy(), + ) + + assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) + assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("BBM_CAIYUN_API_KEY"), + reason="No BBM_CAIYUN_API_KEY in environment variable.", +) +def test_caiyun_translate_txt(test_book_dir, tmpdir): + """Test caiyun tranlate txt""" + shutil.copyfile( + os.path.join(test_book_dir, "the_little_prince.txt"), + os.path.join(tmpdir, "the_little_prince.txt"), + ) + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "the_little_prince.txt"), + "--test", + "--batch_size", + "30", + "--test_num", + "100", + "--model", + "caiyun", + ], + env=os.environ.copy(), + ) + + assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) + assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("BBM_DEEPL_API_KEY"), + reason="No BBM_DEEPL_API_KEY in environment variable.", +) +def test_deepl_translate_txt(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "the_little_prince.txt"), + os.path.join(tmpdir, "the_little_prince.txt"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "the_little_prince.txt"), + "--test", + "--batch_size", + "30", + "--test_num", + "20", + "--model", + "deepl", + ], + env=os.environ.copy(), + ) + + assert os.path.isfile(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) + assert os.path.getsize(os.path.join(tmpdir, "the_little_prince_bilingual.txt")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("BBM_DEEPL_API_KEY"), + reason="No BBM_DEEPL_API_KEY in environment variable.", +) +def test_deepl_translate_srt(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "Lex_Fridman_episode_322.srt"), + os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), + "--test", + "--batch_size", + "30", + "--test_num", + "20", + "--model", + "deepl", + ], + env=os.environ.copy(), + ) + + assert os.path.isfile(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) + assert ( + os.path.getsize(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) + != 0 + ) + + +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="No OPENAI_API_KEY in environment variable.", +) +def test_openai_translate_epub_zh_hans(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "lemo.epub"), + os.path.join(tmpdir, "lemo.epub"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "lemo.epub"), + "--test", + "--test_num", + "5", + "--language", + "zh-hans", + ], + env=os.environ.copy(), + ) + assert os.path.isfile(os.path.join(tmpdir, "lemo_bilingual.epub")) + assert os.path.getsize(os.path.join(tmpdir, "lemo_bilingual.epub")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="No OPENAI_API_KEY in environment variable.", +) +def test_openai_translate_epub_ja_prompt_txt(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "animal_farm.epub"), + os.path.join(tmpdir, "animal_farm.epub"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "animal_farm.epub"), + "--test", + "--test_num", + "5", + "--language", + "ja", + "--model", + "gpt3", + "--prompt", + "prompt_template_sample.txt", + ], + env=os.environ.copy(), + ) + assert os.path.isfile(os.path.join(tmpdir, "animal_farm_bilingual.epub")) + assert os.path.getsize(os.path.join(tmpdir, "animal_farm_bilingual.epub")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="No OPENAI_API_KEY in environment variable.", +) +def test_openai_translate_epub_ja_prompt_json(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "animal_farm.epub"), + os.path.join(tmpdir, "animal_farm.epub"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join(tmpdir, "animal_farm.epub"), + "--test", + "--test_num", + "5", + "--language", + "ja", + "--prompt", + "prompt_template_sample.json", + ], + env=os.environ.copy(), + ) + assert os.path.isfile(os.path.join(tmpdir, "animal_farm_bilingual.epub")) + assert os.path.getsize(os.path.join(tmpdir, "animal_farm_bilingual.epub")) != 0 + + +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="No OPENAI_API_KEY in environment variable.", +) +def test_openai_translate_srt(test_book_dir, tmpdir): + shutil.copyfile( + os.path.join(test_book_dir, "Lex_Fridman_episode_322.srt"), + os.path.join(tmpdir, "Lex_Fridman_episode_322.srt"), + ) + + subprocess.run( + [ + sys.executable, + "make_book.py", + "--book_name", + os.path.join("Lex_Fridman_episode_322.srt"), + "--test", + "--test_num", + "20", + ], + env=os.environ.copy(), + ) + assert os.path.isfile(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) + assert ( + os.path.getsize(os.path.join(tmpdir, "Lex_Fridman_episode_322_bilingual.srt")) + != 0 + )