New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[feat(whisper)] Add recognize_whisper #625
Changes from all commits
282402b
aa09576
65e20dd
68b2438
b3665f4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ | |
|
||
import io | ||
import os | ||
import tempfile | ||
import sys | ||
import subprocess | ||
import wave | ||
|
@@ -1666,6 +1667,44 @@ def recognize_tensorflow(self, audio_data, tensor_graph='tensorflow-data/conv_ac | |
for node_id in top_k: | ||
human_string = self.tflabels[node_id] | ||
return human_string | ||
|
||
def recognize_whisper(self, audio_data, model="base", show_dict=False, load_options=None, language=None, translate=False, **transcribe_options): | ||
""" | ||
Performs speech recognition on ``audio_data`` (an ``AudioData`` instance), using Whisper. | ||
|
||
The recognition language is determined by ``language``, an uncapitalized full language name like "english" or "chinese". See the full language list at https://github.com/openai/whisper/blob/main/whisper/tokenizer.py | ||
|
||
model can be any of tiny, base, small, medium, large, tiny.en, base.en, small.en, medium.en. See https://github.com/openai/whisper for more details. | ||
|
||
If show_dict is true, returns the full dict response from Whisper, including the detected language. Otherwise returns only the transcription. | ||
|
||
You can translate the result to english with Whisper by passing translate=True | ||
|
||
Other values are passed directly to whisper. See https://github.com/openai/whisper/blob/main/whisper/transcribe.py for all options | ||
""" | ||
|
||
assert isinstance(audio_data, AudioData), "Data must be audio data" | ||
import whisper | ||
|
||
if load_options or not hasattr(self, "whisper_model") or self.whisper_model.get(model) is None: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✍️memo: https://docs.python.org/3/reference/expressions.html#boolean-operations
|
||
self.whisper_model = getattr(self, "whisper_model", {}) | ||
self.whisper_model[model] = whisper.load_model(model, **load_options or {}) | ||
|
||
with tempfile.NamedTemporaryFile(suffix=".wav") as f: | ||
f.write(audio_data.get_wav_data()) | ||
f.flush() | ||
result = self.whisper_model[model].transcribe( | ||
f.name, | ||
language=language, | ||
task="translate" if translate else None, | ||
**transcribe_options | ||
ftnext marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) | ||
|
||
if show_dict: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nits: I found Conditional expressions |
||
return result | ||
else: | ||
return result["text"] | ||
|
||
|
||
def recognize_vosk(self, audio_data, language='en'): | ||
from vosk import Model, KaldiRecognizer | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,6 +12,7 @@ def setUp(self): | |
self.AUDIO_FILE_EN = os.path.join(os.path.dirname(os.path.realpath(__file__)), "english.wav") | ||
self.AUDIO_FILE_FR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "french.aiff") | ||
self.AUDIO_FILE_ZH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "chinese.flac") | ||
self.WHISPER_CONFIG = {"temperature": 0} | ||
|
||
def test_sphinx_english(self): | ||
r = sr.Recognizer() | ||
|
@@ -81,6 +82,20 @@ def test_ibm_chinese(self): | |
with sr.AudioFile(self.AUDIO_FILE_ZH) as source: audio = r.record(source) | ||
self.assertEqual(r.recognize_ibm(audio, username=os.environ["IBM_USERNAME"], password=os.environ["IBM_PASSWORD"], language="zh-CN"), u"砸 自己 的 脚 ") | ||
|
||
def test_whisper_english(self): | ||
r = sr.Recognizer() | ||
with sr.AudioFile(self.AUDIO_FILE_EN) as source: audio = r.record(source) | ||
self.assertEqual(r.recognize_whisper(audio, language="english", **self.WHISPER_CONFIG), " 1, 2, 3") | ||
|
||
def test_whisper_french(self): | ||
r = sr.Recognizer() | ||
with sr.AudioFile(self.AUDIO_FILE_FR) as source: audio = r.record(source) | ||
self.assertEqual(r.recognize_whisper(audio, language="french", **self.WHISPER_CONFIG), " et c'est la dictée numéro 1.") | ||
|
||
def test_whisper_chinese(self): | ||
r = sr.Recognizer() | ||
with sr.AudioFile(self.AUDIO_FILE_ZH) as source: audio = r.record(source) | ||
self.assertEqual(r.recognize_whisper(audio, model="small", language="chinese", **self.WHISPER_CONFIG), u"砸自己的腳") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
✍️ When I specify
|
||
|
||
if __name__ == "__main__": | ||
unittest.main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It works!🎉 Thanks.