Preface:
Speech evaluator:
Through intelligent speech technology, it can automatically evaluate the pronunciation level, locate the pronunciation errors and defects, and analyze the problems. At present, the assessment of sound provides the assessment of Chinese and English, and supports three types of questions: single word (Chinese exclusive), word and sentence reading.
Design sketch:
1,IseActivity.java
public class IseActivity extends AppCompatActivity implements View.OnClickListener {
private static final String TAG = "IseActivity";
private EditText mEvaTextEditText;
private EditText mResultEditText;
private Button mIseStartButton;
private Button iseLanguages, iseTopic, iseResultLevel;
private EditText iseStartTime, iseEndTime, iseTime;
private String mLastResult;
private SpeechEvaluator mIse;
//Switch (Chinese, sentence, level)
private String languageType = "zh_cn", topicType = "read_sentence", resultLevelType = "complete";
@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_ise);
mIse = SpeechEvaluator.createEvaluator(this, null);
initUI();
setEvaText();
}
private void initUI() {
mEvaTextEditText = this.findViewById(R.id.ise_eva_text);
mResultEditText = this.findViewById(R.id.ise_result_text);
mIseStartButton = this.findViewById(R.id.ise_start);
iseLanguages = this.findViewById(R.id.ise_languages);
iseTopic = this.findViewById(R.id.ise_topic);
iseResultLevel = this.findViewById(R.id.ise_result_level);
iseStartTime = this.findViewById(R.id.ise_start_time);
iseEndTime = this.findViewById(R.id.ise_end_time);
iseTime = this.findViewById(R.id.ise_time);
mIseStartButton.setOnClickListener(this);
iseLanguages.setOnClickListener(this);
iseTopic.setOnClickListener(this);
iseResultLevel.setOnClickListener(this);
findViewById(R.id.ise_parse).setOnClickListener(this);
findViewById(R.id.ise_stop).setOnClickListener(this);
findViewById(R.id.ise_cancel).setOnClickListener(this);
}
@Override
public void onClick(View view) {
if (null == mIse) {
// Failed to create a single instance, the same reason as 21001 error, refer to http://bbs.xfyun.cn/forum.php? Mod = viewthread & TID = 9688
showToast("Failed to create object, please confirm libmsc.so Placed correctly and called createUtility Initialize");
return;
}
switch (view.getId()) {
case R.id.ise_start:
if (mIse == null) {
return;
}
String evaText = mEvaTextEditText.getText().toString();
mLastResult = null;
mResultEditText.setText("");
mResultEditText.setHint("Please read the above");
mIseStartButton.setEnabled(false);
setParams();
int ret = mIse.startEvaluating(evaText, null, mEvaluatorListener);
//The following method is to evaluate the service by writing audio directly
/*if (ret != ErrorCode.SUCCESS) {
showTip("Recognition failed, error code: "+ RET";
} else {
showTip(getString(R.string.text_begin_ise));
byte[] audioData = FucUtil.readAudioFile(IseDemo.this,"isetest.wav");
if(audioData != null) {
//Prevent writing audio too early to fail
try{
new Thread().sleep(100);
}catch (InterruptedException e) {
Log.d(TAG,"InterruptedException :"+e);
}
mIse.writeAudio(audioData,0,audioData.length);
mIse.stopEvaluating();
}
}*/
break;
case R.id.ise_parse:
// Analyze the final result
if (!TextUtils.isEmpty(mLastResult)) {
XmlResultParser resultParser = new XmlResultParser();
Result result = resultParser.parse(mLastResult);
if (null != result) {
mResultEditText.setText(result.toString());
} else {
showToast("Resolution result is empty");
}
}
break;
case R.id.ise_stop:
if (mIse.isEvaluating()) {
mResultEditText.setHint("Evaluation stopped, waiting for results...");
mIse.stopEvaluating();
}
break;
case R.id.ise_cancel:
mIse.cancel();
mIseStartButton.setEnabled(true);
mResultEditText.setText("");
mResultEditText.setHint("Please click the "start evaluation" button");
mLastResult = null;
break;
case R.id.ise_languages:
if (languageType.equals("zh_cn")) {
languageType = "en_us";
iseLanguages.setText("Language: English");
} else {
languageType = "zh_cn";
iseLanguages.setText("Language: Chinese");
}
setEvaText();
break;
case R.id.ise_topic:
if (topicType.equals("read_sentence")) {
topicType = "read_word";
iseTopic.setText("Question type: words");
} else if (topicType.equals("read_word")) {
topicType = "read_syllable";
iseTopic.setText("Question type: words");
} else {
topicType = "read_sentence";
iseTopic.setText("Question type: sentence");
}
setEvaText();
break;
case R.id.ise_result_level:
if (resultLevelType.equals("complete")) {
resultLevelType = "plain";
iseResultLevel.setText("Grade: plain");
} else {
resultLevelType = "complete";
iseResultLevel.setText("Grade: complete");
}
setEvaText();
break;
default:
}
}
// Set up evaluation questions
private void setEvaText() {
String text = "";
if ("en_us".equals(languageType)) {
if ("read_word".equals(topicType)) {
text = getString(R.string.text_en_word);
} else if ("read_sentence".equals(topicType)) {
text = getString(R.string.text_en_sentence);
}
} else {
// Chinese evaluation
if ("read_syllable".equals(topicType)) {
text = getString(R.string.text_cn_syllable);
} else if ("read_word".equals(topicType)) {
text = getString(R.string.text_cn_word);
} else if ("read_sentence".equals(topicType)) {
text = getString(R.string.text_cn_sentence);
}
}
mEvaTextEditText.setText(text);
mResultEditText.setText("");
mLastResult = null;
mResultEditText.setHint("Please click the "start evaluation" button");
}
private void setParams() {
// Set up profiling language
mIse.setParameter(SpeechConstant.LANGUAGE, languageType);
// Set the type to be measured
mIse.setParameter(SpeechConstant.ISE_CATEGORY, topicType);
// Set result level (Chinese only supports complete)
mIse.setParameter(SpeechConstant.RESULT_LEVEL, resultLevelType);
mIse.setParameter(SpeechConstant.TEXT_ENCODING, "utf-8");
// Set the voice front endpoint: Mute timeout, that is, how long the user does not speak is treated as timeout
mIse.setParameter(SpeechConstant.VAD_BOS, iseStartTime.getText().toString());
// Set the end point after voice: the mute detection time of the back-end point, that is, how long does the user stop talking, that is, no longer input, and automatically stop recording
mIse.setParameter(SpeechConstant.VAD_EOS, iseEndTime.getText().toString());
// Voice input timeout, that is, how long the user can say continuously at most; (- 1 no timeout)
mIse.setParameter(SpeechConstant.KEY_SPEECH_TIMEOUT, iseTime.getText().toString());
mIse.setParameter(SpeechConstant.AUDIO_FORMAT_AUE, "opus");
// Set the audio saving path, save the audio format to support pcm and wav, set the path to sd card, please note the write ﹣ external ﹣ storage permission
mIse.setParameter(SpeechConstant.AUDIO_FORMAT, "wav");
mIse.setParameter(SpeechConstant.ISE_AUDIO_PATH, Environment.getExternalStorageDirectory().getAbsolutePath() + "/msc/helloword_ise.wav");
//This setting is required when writing audio directly through writeaudio
//mIse.setParameter(SpeechConstant.AUDIO_SOURCE,"-1");
}
// Evaluation and monitoring interface
private EvaluatorListener mEvaluatorListener = new EvaluatorListener() {
@Override
public void onResult(EvaluatorResult result, boolean isLast) {
Log.e(TAG, "evaluator result :" + isLast);
if (isLast) {
StringBuilder builder = new StringBuilder();
builder.append(result.getResultString());
if (!TextUtils.isEmpty(builder)) {
mResultEditText.setText(builder.toString());
}
mIseStartButton.setEnabled(true);
mLastResult = builder.toString();
showToast("End of evaluation");
}
}
@Override
public void onError(SpeechError error) {
mIseStartButton.setEnabled(true);
if (error != null) {
showToast("error:" + error.getErrorCode() + "," + error.getErrorDescription());
mResultEditText.setText("");
mResultEditText.setHint("Please click the "start evaluation" button");
} else {
Log.e(TAG, "evaluator over");
}
}
@Override
public void onBeginOfSpeech() {
// This callback indicates that the sdk internal recorder is ready and the user can start voice input
Log.e(TAG, "onBeginOfSpeech: evaluator begin");
}
@Override
public void onEndOfSpeech() {
// This callback indicates that the end point of the voice has been detected, and it has entered the recognition process and no longer accepts the voice input
Log.e(TAG, "onEndOfSpeech: evaluator stoped");
}
@Override
public void onVolumeChanged(int volume, byte[] data) {
showToast("Current volume:" + volume);
Log.e(TAG, "onVolumeChanged: Return audio data" + data.length);
}
@Override
public void onEvent(int eventType, int arg1, int arg2, Bundle obj) {
// The following code is used to obtain the session id with the cloud. When there is a business error, the session id is provided to the technical support personnel, which can be used to query the session log and locate the reason for the error
// if (SpeechEvent.EVENT_SESSION_ID == eventType) {
// String sid = obj.getString(SpeechEvent.KEY_EVENT_SESSION_ID);
// Log.d(TAG, "session id =" + sid);
// }
}
};
/**
* Show toast
*/
private void showToast(final String str) {
Toast.makeText(this, str, Toast.LENGTH_SHORT).show();
}
@Override
protected void onResume() {
// Open statistics mobile data statistical analysis
//FlowerCollector.onResume(IseDemo.this);
//FlowerCollector.onPageStart(TAG);
super.onResume();
}
@Override
protected void onPause() {
// Open statistics mobile data statistical analysis
//FlowerCollector.onPageEnd(TAG);
//FlowerCollector.onPause(IseDemo.this);
super.onPause();
}
@Override
protected void onDestroy() {
super.onDestroy();
if (null != mIse) {
mIse.destroy();
mIse = null;
}
}
}
Layout file activity · ise.xml:
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:gravity="center_horizontal"
android:orientation="vertical"
android:paddingLeft="10dp"
android:paddingRight="10dp">
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_centerInParent="true"
android:text="IFLYTEK evaluation example"
android:textSize="30sp" />
<EditText
android:id="@+id/ise_eva_text"
android:layout_width="match_parent"
android:layout_height="120dp"
android:layout_marginBottom="5dp"
android:gravity="top|left"
android:textSize="20sp" />
<EditText
android:id="@+id/ise_result_text"
android:layout_width="match_parent"
android:layout_height="0dp"
android:layout_weight="1"
android:gravity="top|left"
android:hint="Please click the "start evaluation" button"
android:textSize="16sp" />
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="10dp"
android:gravity="center_vertical"
android:orientation="horizontal">
<Button
android:id="@+id/ise_languages"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Language: Chinese"
android:textSize="16sp" />
<Button
android:id="@+id/ise_topic"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Question type: sentence"
android:textSize="16sp" />
<Button
android:id="@+id/ise_result_level"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Grade: complete"
android:textSize="16sp" />
</LinearLayout>
<EditText
android:id="@+id/ise_start_time"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:hint="Front end timeout:(Default: 5000 ms)"
android:inputType="number"
android:padding="10dp"
android:textSize="20sp" />
<EditText
android:id="@+id/ise_end_time"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:hint="Backend timeout:(Default: 1800 ms)"
android:inputType="number"
android:padding="10dp"
android:textSize="20sp" />
<EditText
android:id="@+id/ise_time"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:hint="Evaluation timeout:(Acquiescence:-1 No timeout, same as above)"
android:inputType="number"
android:padding="10dp"
android:textSize="20sp" />
<TextView
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="You need to start the evaluation again after modifying the value"
android:textSize="15sp" />
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="5dp"
android:layout_marginBottom="2dp"
android:gravity="center_horizontal"
android:orientation="horizontal">
<Button
android:id="@+id/ise_start"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Start evaluation"
android:textSize="20sp" />
<Button
android:id="@+id/ise_stop"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Stop evaluation"
android:textSize="20sp" />
</LinearLayout>
<LinearLayout
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginTop="10dp"
android:layout_marginBottom="2dp"
android:gravity="center_horizontal"
android:orientation="horizontal">
<Button
android:id="@+id/ise_cancel"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Cancel evaluation"
android:textSize="20sp" />
<Button
android:id="@+id/ise_parse"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_weight="1"
android:text="Result analysis"
android:textSize="20sp" />
</LinearLayout>
</LinearLayout>
2. Tool class XmlResultParser.java (also available in Demo):
public class XmlResultParser {
public Result parse(String xml) {
if (TextUtils.isEmpty(xml)) {
return null;
}
XmlPullParser pullParser = Xml.newPullParser();
try {
InputStream ins = new ByteArrayInputStream(xml.getBytes());
pullParser.setInput(ins, "utf-8");
FinalResult finalResult = null;
int eventType = pullParser.getEventType();
while (XmlPullParser.END_DOCUMENT != eventType) {
switch (eventType) {
case XmlPullParser.START_TAG:
if ("FinalResult".equals(pullParser.getName())) {
// There's only one total score
finalResult = new FinalResult();
} else if ("ret".equals(pullParser.getName())) {
finalResult.ret = getInt(pullParser, "value");
} else if ("total_score".equals(pullParser.getName())) {
finalResult.total_score = getFloat(pullParser, "value");
} else if ("xml_result".equals(pullParser.getName())) {
// Detailed results
return parseResult(pullParser);
}
break;
case XmlPullParser.END_TAG:
if ("FinalResult".equals(pullParser.getName())) {
return finalResult;
}
break;
default:
break;
}
eventType = pullParser.next();
}
} catch (XmlPullParserException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private Result parseResult(XmlPullParser pullParser) {
Result result = null;
// Whether the < rec < paper > tag has been scanned to
boolean rec_paperPassed = false;
Sentence sentence = null;
Word word = null;
Syll syll = null;
Phone phone = null;
int eventType;
try {
eventType = pullParser.getEventType();
while (XmlPullParser.END_DOCUMENT != eventType) {
switch (eventType) {
case XmlPullParser.START_TAG:
if ("rec_paper".equals(pullParser.getName())) {
rec_paperPassed = true;
} else if ("read_syllable".equals(pullParser.getName())) {
if (!rec_paperPassed) {
result = new ReadSyllableResult();
} else {
readTotalResult(result, pullParser);
}
} else if ("read_word".equals(pullParser.getName())) {
if (!rec_paperPassed) {
result = new ReadWordResult();
String lan = getLanguage(pullParser);
result.language = (null == lan) ? "cn" : lan;
} else {
readTotalResult(result, pullParser);
}
} else if ("read_sentence".equals(pullParser.getName()) ||
"read_chapter".equals(pullParser.getName())) {
if (!rec_paperPassed) {
result = new ReadSentenceResult();
String lan = getLanguage(pullParser);
result.language = (null == lan) ? "cn" : lan;
} else {
readTotalResult(result, pullParser);
}
} else if ("sentence".equals(pullParser.getName())) {
if (null == result.sentences) {
result.sentences = new ArrayList<Sentence>();
}
sentence = createSentence(pullParser);
} else if ("word".equals(pullParser.getName())) {
if (null != sentence && null == sentence.words) {
sentence.words = new ArrayList<Word>();
}
word = createWord(pullParser);
} else if ("syll".equals(pullParser.getName())) {
if (null != word && null == word.sylls) {
word.sylls = new ArrayList<Syll>();
}
syll = createSyll(pullParser);
} else if ("phone".equals(pullParser.getName())) {
if (null != syll && null == syll.phones) {
syll.phones = new ArrayList<Phone>();
}
phone = createPhone(pullParser);
}
break;
case XmlPullParser.END_TAG:
if ("phone".equals(pullParser.getName())) {
syll.phones.add(phone);
} else if ("syll".equals(pullParser.getName())) {
word.sylls.add(syll);
} else if ("word".equals(pullParser.getName())) {
sentence.words.add(word);
} else if ("sentence".equals(pullParser.getName())) {
result.sentences.add(sentence);
} else if ("read_syllable".equals(pullParser.getName())
|| "read_word".equals(pullParser.getName())
|| "read_sentence".equals(pullParser.getName())) {
return result;
}
break;
default:
break;
}
eventType = pullParser.next();
}
} catch (XmlPullParserException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
private void readTotalResult(Result result, XmlPullParser pullParser) {
result.beg_pos = getInt(pullParser, "beg_pos");
result.end_pos = getInt(pullParser, "end_pos");
result.content = getContent(pullParser);
result.total_score = getFloat(pullParser, "total_score");
result.time_len = getInt(pullParser, "time_len");
result.except_info = getExceptInfo(pullParser);
result.is_rejected = getIsRejected(pullParser);
}
private Phone createPhone(XmlPullParser pullParser) {
Phone phone;
phone = new Phone();
phone.beg_pos = getInt(pullParser, "beg_pos");
phone.end_pos = getInt(pullParser, "end_pos");
phone.content = getContent(pullParser);
phone.dp_message = getInt(pullParser, "dp_message");
phone.time_len = getInt(pullParser, "time_len");
return phone;
}
private Syll createSyll(XmlPullParser pullParser) {
Syll syll;
syll = new Syll();
syll.beg_pos = getInt(pullParser, "beg_pos");
syll.end_pos = getInt(pullParser, "end_pos");
syll.content = getContent(pullParser);
syll.symbol = getSymbol(pullParser);
syll.dp_message = getInt(pullParser, "dp_message");
syll.time_len = getInt(pullParser, "time_len");
return syll;
}
private Word createWord(XmlPullParser pullParser) {
Word word;
word = new Word();
word.beg_pos = getInt(pullParser, "beg_pos");
word.end_pos = getInt(pullParser, "end_pos");
word.content = getContent(pullParser);
word.symbol = getSymbol(pullParser);
word.time_len = getInt(pullParser, "time_len");
word.dp_message = getInt(pullParser, "dp_message");
word.total_score = getFloat(pullParser, "total_score");
word.global_index = getInt(pullParser, "global_index");
word.index = getInt(pullParser, "index");
return word;
}
private Sentence createSentence(XmlPullParser pullParser) {
Sentence sentence;
sentence = new Sentence();
sentence.beg_pos = getInt(pullParser, "beg_pos");
sentence.end_pos = getInt(pullParser, "end_pos");
sentence.content = getContent(pullParser);
sentence.time_len = getInt(pullParser, "time_len");
sentence.index = getInt(pullParser, "index");
sentence.word_count = getInt(pullParser, "word_count");
return sentence;
}
private String getLanguage(XmlPullParser pullParser) {
return pullParser.getAttributeValue(null, "lan");
}
private String getExceptInfo(XmlPullParser pullParser) {
return pullParser.getAttributeValue(null, "except_info");
}
private boolean getIsRejected(XmlPullParser pullParser) {
String isRejected = pullParser.getAttributeValue(null, "is_rejected");
if (null == isRejected) {
return false;
}
return Boolean.parseBoolean(isRejected);
}
private String getSymbol(XmlPullParser pullParser) {
return pullParser.getAttributeValue(null, "symbol");
}
private float getFloat(XmlPullParser pullParser, String attrName) {
String val = pullParser.getAttributeValue(null, attrName);
if (null == val) {
return 0f;
}
return Float.parseFloat(val);
}
private String getContent(XmlPullParser pullParser) {
return pullParser.getAttributeValue(null, "content");
}
private int getInt(XmlPullParser pullParser, String attrName) {
String val = pullParser.getAttributeValue(null, attrName);
if (null == val) {
return 0;
}
return Integer.parseInt(val);
}
}
Copy the result folder in Demo to the project, as shown in the following figure:
3. strings.xml used:
<string name="text_en_word">"[word]\napple\nbanana\norange"</string> <string name="text_en_sentence">"The quick brown fox jumps over the lazy dog."</string> <string name="text_cn_syllable">"Know, crazy, yes"</string> <string name="text_cn_word">"Magnet, lead, fragile, hands on, guzheng"</string> <string name="text_cn_sentence">"A snow peak into the sky, the summit silver glittering, large and small lakes, like many jewels inlaid in the ribbon like valley."</string>
