diff --git a/.github/workflows/make_test_ebook.yaml b/.github/workflows/make_test_ebook.yaml index fd41bf6..909d8d8 100644 --- a/.github/workflows/make_test_ebook.yaml +++ b/.github/workflows/make_test_ebook.yaml @@ -10,6 +10,7 @@ on: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + BBM_CAIYUN_API_KEY: ${{ secrets.BBM_CAIYUN_API_KEY }} BBM_DEEPL_API_KEY: ${{ secrets.BBM_DEEPL_API_KEY }} jobs: @@ -33,8 +34,41 @@ jobs: run: | pip install . - - name: Run tests + - name: make normal ebook test using google translate and cli run: | - pip install pytest - pytest tests + bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 10 --model google --translate-tags div,p + bbook_maker --book_name "test_books/Liber_Esther.epub" --test --test_num 20 --model google + - name: make txt book test using google translate + run: | + python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --test_num 20 --model google + + - name: make txt book test with batch_size + run: | + python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 20 --model google + + - name: make caiyun translator test + if: env.BBM_CAIYUN_API_KEY != null + run: | + python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 100 --model caiyun + + - name: make deepl translator test + if: env.BBM_CAIYUN_API_KEY != null + run: | + python3 make_book.py --book_name "test_books/the_little_prince.txt" --test --batch_size 30 --test_num 20 --model deepl + python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test + + - name: make openai key ebook test + if: env.BBM_DEEPL_API_KEY != null + run: | + python3 make_book.py --book_name "test_books/lemo.epub" --test --test_num 5 --language zh-hans + python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --model gpt3 --prompt prompt_template_sample.txt + python3 make_book.py --book_name "test_books/animal_farm.epub" --test --test_num 5 --language ja --prompt prompt_template_sample.json + python3 make_book.py --book_name test_books/Lex_Fridman_episode_322.srt --test --test_num 20 + + - name: Rename and Upload ePub + if: env.OPENAI_API_KEY != null + uses: actions/upload-artifact@v2 + with: + name: epub_output + path: "test_books/lemo_bilingual.epub" diff --git a/book_maker/loader/epub_loader.py b/book_maker/loader/epub_loader.py index 56f8dae..ce93ed5 100644 --- a/book_maker/loader/epub_loader.py +++ b/book_maker/loader/epub_loader.py @@ -128,6 +128,9 @@ class EPUBBookLoader(BaseBookLoader): new_p = copy(p) for p_exclude in self.exclude_translate_tags.split(","): + # for issue #280 + if type(p) == NavigableString: + continue for pt in new_p.find_all(p_exclude): pt.extract() @@ -159,6 +162,9 @@ class EPUBBookLoader(BaseBookLoader): temp_p = copy(p) for p_exclude in self.exclude_translate_tags.split(","): + # for issue #280 + if type(p) == NavigableString: + continue for pt in temp_p.find_all(p_exclude): pt.extract() @@ -276,8 +282,9 @@ class EPUBBookLoader(BaseBookLoader): break for t in extract_p_list_ori: - target.insert_after(t) - target = t + if target: + target.insert_after(t) + target = t for item in complete_book.get_items(): if item.file_name != fixname: diff --git a/book_maker/translator/caiyun_translator.py b/book_maker/translator/caiyun_translator.py index 09bbad6..34be507 100644 --- a/book_maker/translator/caiyun_translator.py +++ b/book_maker/translator/caiyun_translator.py @@ -1,5 +1,6 @@ import json import re +import time import requests from rich import print @@ -43,6 +44,20 @@ class Caiyun(Base): data=json.dumps(payload), headers=self.headers, ) - t_text = json.loads(response.text)["target"] + try: + t_text = response.json()["target"] + except Exception as e: + print(str(e), response.text, "will sleep 60s for the time limit") + if "limit" in response.json()["message"]: + print("will sleep 60s for the time limit") + time.sleep(60) + response = requests.request( + "POST", + self.api_url, + data=json.dumps(payload), + headers=self.headers, + ) + t_text = response.json()["target"] + print("[bold green]" + re.sub("\n{3,}", "\n\n", t_text) + "[/bold green]") return t_text