From 6e22cfb7df42f161b0c705465220fb840039c441 Mon Sep 17 00:00:00 2001 From: guaguaguaxia Date: Sat, 4 Mar 2023 18:30:59 +0800 Subject: [PATCH] =?UTF-8?q?feat:multi=20apikey=20to=20break=20through=20th?= =?UTF-8?q?e=20limitation,and=20fix=20some=20common=E2=80=A6=20(#14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat:multi apikey to break through the limitation,and fix some common english mistakes --------- Co-authored-by: yihong0618 --- README.md | 7 ++++--- make_book.py | 50 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 379e4bb..42c646b 100644 --- a/README.md +++ b/README.md @@ -15,14 +15,15 @@ Make bilingual epub books Using AI translate ## 使用 1. pip install -r requirements.txt -2. openapi token +2. OpenAI API key，如果有多个可以用英文逗号分隔(xxx,xxx,xxx)，可以减少接口调用次数限制带来的错误 3. 本地放了一个 animal_farm.epub 给大家测试 4. 默认用了 ChatGPT 模型，用 `--model gpt3` 来使用 gpt3 模型 5. 加了 `--test` 命令如果大家没付费可以加上这个先看看效果（有 limit 稍微有些慢） 6. Set the target language like `--language "Simplified Chinese"`. Suppot ` "Japanese" / "Traditional Chinese" / "German" / "French" / "Korean"`. Default target language is `"Simplified Chinese"`. Support language list please see the LANGUAGES at [utils.py](./utils.py). -7. 加入 `--resume` 命令，可以手动中断后，加入命令继续执行。 +7. 加了 `--proxy` 参数，方便中国大陆的用户在本地测试时使用代理，传入类似 `http://127.0.0.1:7890` 的字符串 +8. 加入 `--resume` 命令，可以手动中断后，加入命令继续执行。 e.g. ```shell @@ -39,7 +40,7 @@ python3 make_book.py --book_name test_books/animal_farm.epub --model gpt3 --no_l 1. 有 limit 如果想要速度可以付费 2. 现在是 demo 版本有很多工作要做 PR welcome -3. 尤其是 batch translat 做完效果会好很多 +3. 尤其是 batch translate 做完效果会好很多 4. DeepL 模型稍后更新 diff --git a/make_book.py b/make_book.py index d679ea6..98bff54 100644 --- a/make_book.py +++ b/make_book.py @@ -1,4 +1,6 @@ import argparse +import os +import random import pickle import time from abc import abstractmethod @@ -20,7 +22,15 @@ RESUME = False class Base: def __init__(self, key, language): - pass + self.key = key + self.language =language + self.current_key_index = 0 + + def get_key(self, key_str): + keys = key_str.split(",") + key = keys[self.current_key_index] + self.current_key_index = (self.current_key_index + 1) % len(keys) + return key @abstractmethod def translate(self, text): @@ -29,11 +39,11 @@ class Base: class GPT3(Base): def __init__(self, key, language): + super().__init__(key, language) self.api_key = key self.api_url = "https://api.openai.com/v1/completions" self.headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}", } # TODO support more models here self.data = { @@ -48,6 +58,7 @@ class GPT3(Base): def translate(self, text): print(text) + self.headers["Authorization"] = f"Bearer {self.get_key(self.api_key)}" self.data["prompt"] = f"Please help me to translate，`{text}` to {self.language}" r = self.session.post(self.api_url, headers=self.headers, json=self.data) if not r.ok: @@ -73,7 +84,7 @@ class ChatGPT(Base): def translate(self, text): print(text) - openai.api_key = self.key + openai.api_key = self.get_key(self.key) try: completion = openai.ChatCompletion.create( model="gpt-3.5-turbo", @@ -81,7 +92,7 @@ class ChatGPT(Base): { "role": "user", # english prompt here to save tokens - "content": f"Please help me to translate，`{text}` to {self.language}, please return only translated content not include the origin text", + "content": f"Please help me to translate,`{text}` to {self.language}, please return only translated content not include the origin text", } ], ) @@ -96,9 +107,12 @@ class ChatGPT(Base): # for time limit time.sleep(3) except Exception as e: - print(str(e), "will sleep 60 seconds") # TIME LIMIT for open api please pay - time.sleep(60) + key_len = self.key.count(",") + 1 + sleep_time = int(60 / key_len) + time.sleep(sleep_time) + print(str(e), "will sleep " + str(sleep_time) + " seconds") + openai.api_key = self.get_key(self.key) completion = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ @@ -201,20 +215,21 @@ if __name__ == "__main__": "--book_name", dest="book_name", type=str, - help="your epub book name", + help="your epub book file path", ) parser.add_argument( "--openai_key", dest="openai_key", type=str, default="", - help="openai api key", + help="openai api key,if you have more than one key,you can use comma" + " to split them and you can break through the limitation", ) parser.add_argument( "--no_limit", dest="no_limit", action="store_true", - help="if you pay add it", + help="If you are a paying customer you can add it", ) parser.add_argument( "--test", @@ -229,7 +244,6 @@ if __name__ == "__main__": default=10, help="test num for the test", ) - parser.add_argument( "-m", "--model", @@ -237,7 +251,7 @@ if __name__ == "__main__": type=str, default="chatgpt", choices=["chatgpt", "gpt3"], # support DeepL later - help="Use which model", + help="Which model to use", ) parser.add_argument( "--language", @@ -253,10 +267,24 @@ if __name__ == "__main__": action="store_true", help="if program accidentally stop you can use this to resume", ) + parser.add_argument( + "-p", + "--proxy", + dest="proxy", + type=str, + default="", + help="use proxy like http://127.0.0.1:7890", + ) + options = parser.parse_args() NO_LIMIT = options.no_limit IS_TEST = options.test TEST_NUM = options.test_num + PROXY = options.proxy + if PROXY != "": + os.environ["http_proxy"] = PROXY + os.environ["https_proxy"] = PROXY + OPENAI_API_KEY = options.openai_key or env.get("OPENAI_API_KEY") RESUME = options.resume if not OPENAI_API_KEY: