diff --git a/jigasi-home/sip-communicator.properties b/jigasi-home/sip-communicator.properties index 72857f7c5..89c27a450 100644 --- a/jigasi-home/sip-communicator.properties +++ b/jigasi-home/sip-communicator.properties @@ -122,6 +122,7 @@ org.jitsi.jigasi.xmpp.acc.VIDEO_CALLING_DISABLED=true # translation # org.jitsi.jigasi.transcription.ENABLE_TRANSLATION=false +# org.jitsi.jigasi.transcription.SAVE_TRANSLATED_TRANSCRIPTS=false # record audio. Currently only wav format is supported # org.jitsi.jigasi.transcription.RECORD_AUDIO=false diff --git a/src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptPublisher.java b/src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptPublisher.java index fc5d70e3c..70d0f3858 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptPublisher.java +++ b/src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptPublisher.java @@ -441,6 +441,14 @@ protected String getPathsToScriptsToExecuteSeparator() */ protected abstract T formatSpeechEvent(SpeechEvent e); + /** + * Format a translated speech event to the used format + * + * @param e the speech event + * @return the TranslatedSpeechEvent formatted in the desired type + */ + protected abstract T formatTranslatedSpeechEvent(TranslatedSpeechEvent e); + /** * Format a join event to the used format * @@ -480,6 +488,12 @@ public abstract class BaseFormatter * The instant when the conference ended */ protected Instant endInstant; + + /** + * A string of the target language + */ + protected String language; + /** * A string of the room name */ @@ -549,6 +563,20 @@ BaseFormatter tookPlaceAtUrl(String url) return this; } + /** + * Format a transcript in a particular translated language + * + * @param language of translation for this translated transcript + * @return this formatter + */ + BaseFormatter transcriptLanguage(String language) + { + if(language != null) + { + this.language = language; + } + return this; + } /** * Format a transcript which includes the list of initial participant * @@ -581,6 +609,26 @@ BaseFormatter speechEvents(List events) return this; } + /** + * Format a transcript which includes the translations of what everyone + * who was transcribed said. Ignored when the given event does not have + * the event type {@link Transcript.TranscriptEventType#SPEECH} + * + * @param events a list of events containing the translated transcriptions + * @return this formatter + */ + BaseFormatter translatedSpeechEvents(List events) + { + for(TranslatedSpeechEvent e : events) + { + if(e.getEvent().equals(Transcript.TranscriptEventType.SPEECH)) + { + formattedEvents.put(e, formatTranslatedSpeechEvent(e)); + } + } + return this; + } + /** * Format a transcript which includes when anyone joined the conference. * Ignored when the given event does not have the event type diff --git a/src/main/java/org/jitsi/jigasi/transcription/LocalJsonTranscriptHandler.java b/src/main/java/org/jitsi/jigasi/transcription/LocalJsonTranscriptHandler.java index b599cee67..14c418869 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/LocalJsonTranscriptHandler.java +++ b/src/main/java/org/jitsi/jigasi/transcription/LocalJsonTranscriptHandler.java @@ -69,6 +69,12 @@ public class LocalJsonTranscriptHandler public final static String JSON_KEY_FINAL_TRANSCRIPT_ROOM_URL = "room_url"; + /** + * This field stores the language in which the transcript is being stored. + */ + public final static String JSON_KEY_FINAL_TRANSCRIPT_LANGUAGE + = "transcript_language"; + /** * This field stores all the events as an JSON array */ @@ -147,9 +153,10 @@ public class LocalJsonTranscriptHandler // "alternative" JSON object fields /** - * This field stores the text of a speech-to-text result as a string + * This field stores the text of a speech-to-text result in original or + * translated language as a string */ - public final static String JSON_KEY_ALTERNATIVE_TEXT = "text"; + public final static String JSON_KEY_TEXT = "text"; /** * This fields stores the confidence of the speech-to-text result as a @@ -275,12 +282,7 @@ private static JSONObject createTranslationJSONObject( result.getTranscriptionResult()); addEventDescriptions(eventObject, event); - - eventObject.put(JSON_KEY_TYPE, JSON_VALUE_TYPE_TRANSLATION_RESULT); - eventObject.put(JSON_KEY_EVENT_LANGUAGE, result.getLanguage()); - eventObject.put(JSON_KEY_ALTERNATIVE_TEXT, result.getTranslatedText()); - eventObject.put(JSON_KEY_EVENT_MESSAGE_ID, - result.getTranscriptionResult().getMessageID().toString()); + addTranslations(eventObject, result); return eventObject; } @@ -299,6 +301,15 @@ protected JSONObject formatSpeechEvent(SpeechEvent e) return object; } + @Override + protected JSONObject formatTranslatedSpeechEvent(TranslatedSpeechEvent e) + { + JSONObject object = new JSONObject(); + addEventDescriptions(object, e); + addTranslations(object, e.getResult()); + return object; + } + @Override protected JSONObject formatJoinEvent(TranscriptEvent e) { @@ -347,7 +358,7 @@ public static void addEventDescriptions( /** * Make a given JSON object the "event" json object by adding the fields - * transcripts, is_interim, messageID and langiage to the given object. + * transcripts, is_interim, messageID and language to the given object. * Assumes that * {@link this#addEventDescriptions(JSONObject, TranscriptEvent)} * has been or will be called on the same given JSON object @@ -365,7 +376,7 @@ private static void addAlternatives(JSONObject jsonObject, SpeechEvent e) { JSONObject alternativeJSON = new JSONObject(); - alternativeJSON.put(JSON_KEY_ALTERNATIVE_TEXT, + alternativeJSON.put(JSON_KEY_TEXT, alternative.getTranscription()); alternativeJSON.put(JSON_KEY_ALTERNATIVE_CONFIDENCE, alternative.getConfidence()); @@ -381,6 +392,25 @@ private static void addAlternatives(JSONObject jsonObject, SpeechEvent e) jsonObject.put(JSON_KEY_EVENT_STABILITY, result.getStability()); } + /** + * Adds event information to the json object by adding the type, + * translated text, is_interim, messageID and language to the given object. + * + * @param object the json object where the required fields are to be added. + * @param result the translation result which contains the required + * language and translated text. + */ + @SuppressWarnings("unchecked") + private static void addTranslations(JSONObject object, + TranslationResult result) + { + object.put(JSON_KEY_TYPE, JSON_VALUE_TYPE_TRANSLATION_RESULT); + object.put(JSON_KEY_EVENT_LANGUAGE, result.getLanguage()); + object.put(JSON_KEY_TEXT, result.getTranslatedText()); + object.put(JSON_KEY_EVENT_MESSAGE_ID, + result.getTranscriptionResult().getMessageID().toString()); + } + /** * Make a given JSON object the "participant" JSON object @@ -446,6 +476,7 @@ private void addTranscriptDescription(JSONObject jsonObject, String roomName, String roomUrl, Collection participants, + String language, Instant start, Instant end, Collection events) @@ -458,6 +489,10 @@ private void addTranscriptDescription(JSONObject jsonObject, { jsonObject.put(JSON_KEY_FINAL_TRANSCRIPT_ROOM_URL, roomUrl); } + if(language != null && !language.isEmpty()) + { + jsonObject.put(JSON_KEY_FINAL_TRANSCRIPT_LANGUAGE, language); + } if(start != null) { jsonObject.put(JSON_KEY_FINAL_TRANSCRIPT_START_TIME, @@ -510,6 +545,7 @@ public JSONObject finish() super.roomName, super.roomUrl, super.initialMembers, + super.language, super.startInstant, super.endInstant, super.getSortedEvents()); @@ -539,6 +575,20 @@ protected void doPublish(Transcript transcript) saveTranscriptStringToFile(getDirPath(), fileName, t.toString()); + + Set targetLanguages = transcript.getTranslationLanguages(); + + targetLanguages.forEach(language -> + { + String translatedTranscript + = transcript.getTranslatedTranscript( + LocalJsonTranscriptHandler.this, language).toString(); + String translationFileName = generateHardToGuessTimeString( + "transcript_" + language, ".json"); + + saveTranscriptStringToFile(getDirPath(), + translationFileName, translatedTranscript); + }); } /** diff --git a/src/main/java/org/jitsi/jigasi/transcription/LocalTxtTranscriptHandler.java b/src/main/java/org/jitsi/jigasi/transcription/LocalTxtTranscriptHandler.java index f23e16512..205448464 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/LocalTxtTranscriptHandler.java +++ b/src/main/java/org/jitsi/jigasi/transcription/LocalTxtTranscriptHandler.java @@ -240,6 +240,23 @@ protected String formatSpeechEvent(SpeechEvent e) + NEW_LINE; } + @Override + protected String formatTranslatedSpeechEvent(TranslatedSpeechEvent e) + { + String name = e.getName(); + String timeStamp = timeFormatter.format(e.getTimeStamp()); + String translatedText = e.getResult().getTranslatedText(); + + String base = String.format(UNFORMATTED_EVENT_BASE, timeStamp, name); + String speech = String.format(UNFORMATTED_SPEECH, translatedText); + String formatted + = base + String.format(UNFORMATTED_SPEECH, translatedText); + + return formatToMaximumLineLength(formatted, MAX_LINE_WIDTH, + base.length() + (speech.length() - translatedText.length())) + + NEW_LINE; + } + @Override protected String formatJoinEvent(TranscriptEvent e) { @@ -462,6 +479,20 @@ protected void doPublish(Transcript transcript) transcript.getTranscript(LocalTxtTranscriptHandler.this); saveTranscriptStringToFile(getDirPath(), fileName, t); + + Set targetLanguages = transcript.getTranslationLanguages(); + + targetLanguages.forEach(language -> + { + String translatedTranscript + = transcript.getTranslatedTranscript( + LocalTxtTranscriptHandler.this, language); + String translationFileName = generateHardToGuessTimeString( + "transcript_" + language, ".txt"); + + saveTranscriptStringToFile(getDirPath(), + translationFileName, translatedTranscript); + }); } /** diff --git a/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java b/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java index c0ffc9cb8..01a1cc512 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java +++ b/src/main/java/org/jitsi/jigasi/transcription/Transcriber.java @@ -66,6 +66,19 @@ public class Transcriber */ public final static boolean ENABLE_TRANSLATION_DEFAULT_VALUE = false; + /** + * The property name for the boolean value whether the translated transcripts + * should be saved + */ + public final static String P_NAME_SAVE_TRANSLATED_TRANSCRIPTS + = "org.jitsi.jigasi.transcription.SAVE_TRANSLATED_TRANSCRIPTS"; + + /** + * Whether to save the transcripts in translated languages. + */ + public final static boolean SAVE_TRANSLATED_TRANSCRIPTS_DEFAULT_VALUE + = false; + /** * The states the transcriber can be in. The Transcriber * can only go through one cycle. So once it is started it can never @@ -132,7 +145,7 @@ private enum State * for managing translations. */ private TranslationManager translationManager - = new TranslationManager(new GoogleCloudTranslationService());; + = new TranslationManager(new GoogleCloudTranslationService()); /** * Every listener which will be notified when a new result comes in @@ -199,6 +212,11 @@ public Transcriber(String roomName, if(isTranslationEnabled()) { addTranscriptionListener(this.translationManager); + if(isSaveTranslatedTranscriptsEnabled()) + { + addTranslationListener(this.transcript); + transcript.setTranslationManager(this.translationManager); + } } this.roomName = roomName; this.roomUrl = roomUrl; @@ -328,7 +346,8 @@ public void updateParticipantSourceLanguage(String identifier, /** * Update the {@link Participant} with the given identifier by setting the * translationLanguage of the participant and update the count for - * languages in the @link {@link TranslationManager} + * languages in the @link {@link TranslationManager} and adds the language + * for the storing the final translated {@link Transcript}. * * @param identifier the identifier of the participant * @param language the language tag to be updated for the participant @@ -345,6 +364,10 @@ public void updateParticipantTargetLanguage(String identifier, translationManager.addLanguage(language); translationManager.removeLanguage(previousLanguage); participant.setTranslationLanguage(language); + if(isSaveTranslatedTranscriptsEnabled()) + { + transcript.addTranslationLanguage(language); + } } } @@ -786,4 +809,16 @@ private boolean isTranslationEnabled() .getBoolean(P_NAME_ENABLE_TRANSLATION, ENABLE_TRANSLATION_DEFAULT_VALUE); } + + /** + * Get whether to store the final transcript in the requested languages. + * + * @return true if enabled, otherwise returns false. + */ + private boolean isSaveTranslatedTranscriptsEnabled() + { + return JigasiBundleActivator.getConfigurationService() + .getBoolean(P_NAME_SAVE_TRANSLATED_TRANSCRIPTS, + SAVE_TRANSLATED_TRANSCRIPTS_DEFAULT_VALUE); + } } diff --git a/src/main/java/org/jitsi/jigasi/transcription/Transcript.java b/src/main/java/org/jitsi/jigasi/transcription/Transcript.java index ccae0f837..7e0fba7ee 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/Transcript.java +++ b/src/main/java/org/jitsi/jigasi/transcription/Transcript.java @@ -22,12 +22,14 @@ /** * A transcript of a conference. An instance of this class will hold the - * complete transcript once a conference is over + * complete transcript once a conference is over in original and language + * for which translations are requested. * * @author Nik Vaessen */ public class Transcript - implements TranscriptionListener + implements TranscriptionListener, + TranslationResultListener { /** * Events which can take place in the transcript @@ -48,6 +50,12 @@ public enum TranscriptEventType */ private final List speechEvents = new LinkedList<>(); + /** + * The map of language tag to the list of speechEvents in that language. + */ + private final Map> translatedSpeechEvents + = new HashMap<>(); + /** * The list of all received */ @@ -89,6 +97,11 @@ public enum TranscriptEventType */ private String roomUrl; + /** + * The translation manager to be used for translating the transcriptions + */ + private TranslationManager translationManager; + /** * Create a Transcript object which can store events related to a conference * which can then be formatted into an actual transcript by a @@ -138,6 +151,55 @@ public void notify(TranscriptionResult result) } } + /** + * Sets the translation manager to be used by this {@link Transcript} + * for translations + * + * @param translationManager to be used + */ + public void setTranslationManager(TranslationManager translationManager) + { + this.translationManager = translationManager; + } + + /** + * Initialises an empty list to hold the Translated results in the given + * language as a key in the map + * + * @param language the target language for creating the final + * translated transcript + */ + public void addTranslationLanguage(String language) + { + if(language == null || language.isEmpty()) + return; + + translatedSpeechEvents.putIfAbsent(language, new LinkedList<>()); + } + + /** + * Returns the list of languages to prepare the final transcript in. + * + * @return set of languages for translations + */ + public Set getTranslationLanguages() + { + return translatedSpeechEvents.keySet(); + } + + @Override + public void notify(TranslationResult result) + { + if(started != null) + { + TranslatedSpeechEvent translatedSpeechEvent + = new TranslatedSpeechEvent(Instant.now(), result); + translatedSpeechEvents.get(result.getLanguage()) + .add(translatedSpeechEvent); + } + } + + @Override public void completed() { @@ -308,18 +370,89 @@ public TranscriptEvent notifyRaisedHand(Participant participant) * @return a formatted transcript with type T */ public T getTranscript(AbstractTranscriptPublisher publisher) + { + return getRoomEvents(publisher) + .speechEvents(speechEvents) + .finish(); + } + + /** + * Get a formatted transcript of events stored by this object in the target + * language. + * + * @param publisher a publisher which has a formatter to create a transcript + * in the desired type. + * @param language the target language of the transcript. + * @param the type in which the transcript will be stored. + * @return a formatted transcript with type T in the target language. + */ + public T getTranslatedTranscript( + AbstractTranscriptPublisher publisher, String language) + { + List translatedSpeechEventsList; + + if(speechEvents.size() != translatedSpeechEvents.get(language).size()) + { + Iterator translatedSpeechEventIterator + = translatedSpeechEvents.get(language).iterator(); + System.out.println(translatedSpeechEvents.get(language)); + translatedSpeechEventsList = new LinkedList<>(); + + TranslatedSpeechEvent currentMessage + = translatedSpeechEventIterator.next(); + + for (SpeechEvent speechEvent : speechEvents) + { + if(currentMessage == null || + currentMessage.getResult().getTranscriptionResult().getMessageID() + != speechEvent.getResult().getMessageID()) + { + // Speech event is not present in the target language + translatedSpeechEventsList.add(new TranslatedSpeechEvent( + speechEvent.getTimeStamp(), + translationManager.getSingleTranslation( + speechEvent.getResult(), language))); + } + else + { + // Speech event is already present in the target language + translatedSpeechEventsList.add(currentMessage); + currentMessage = translatedSpeechEventIterator.hasNext() ? + translatedSpeechEventIterator.next() : null; + } + } + + } + else + { + translatedSpeechEventsList = translatedSpeechEvents.get(language); + } + + return getRoomEvents(publisher) + .transcriptLanguage(language) + .translatedSpeechEvents(translatedSpeechEventsList) + .finish(); + } + + /** + * Gets a formatter of type T with details of the room events. + * + * @param publisher a publisher which has a formatter to create a transcript + * in the desired type. + * @param the type in which the room details are to be formatted. + * @return a formatter for the transcript with type T. + */ + public AbstractTranscriptPublisher.BaseFormatter + getRoomEvents(AbstractTranscriptPublisher publisher) { return publisher.getFormatter() .startedOn(started) .initialParticipants(initialParticipantNames) .tookPlaceInRoom(roomName) .tookPlaceAtUrl(roomUrl) - .speechEvents(speechEvents) .raiseHandEvents(raisedHandEvents) .joinEvents(joinedEvents) .leaveEvents(leftEvents) - .endedOn(ended) - .finish(); + .endedOn(ended); } - } diff --git a/src/main/java/org/jitsi/jigasi/transcription/TranslatedSpeechEvent.java b/src/main/java/org/jitsi/jigasi/transcription/TranslatedSpeechEvent.java new file mode 100644 index 000000000..69a47692e --- /dev/null +++ b/src/main/java/org/jitsi/jigasi/transcription/TranslatedSpeechEvent.java @@ -0,0 +1,60 @@ +/* + * Jigasi, the JItsi GAteway to SIP. + * + * Copyright @ 2017 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.jigasi.transcription; + +import java.time.Instant; + +/** + * A TranslatedSpeechEvent extends a normal TranscriptEvent by including a + * TranslationResult + * + * @author Praveen Kumar Gupta + */ +public class TranslatedSpeechEvent + extends TranscriptEvent +{ + /** + * The transcriptionResult + */ + private TranslationResult result; + + /** + * Create a ResultHolder with a given TranslationResult and timestamp + * + * @param timeStamp the time when the result was received + * @param result the result which was received + */ + TranslatedSpeechEvent(Instant timeStamp, TranslationResult result) + { + super(timeStamp, result.getTranscriptionResult().getParticipant(), + Transcript.TranscriptEventType.SPEECH); + this.result = result; + } + + /** + * Get the TranslationResult this holder is holding + * + * @return the result + */ + public TranslationResult getResult() + { + return result; + } + +} + diff --git a/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java b/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java index 005f1d5f7..5f4d0bf67 100644 --- a/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java +++ b/src/main/java/org/jitsi/jigasi/transcription/TranslationManager.java @@ -114,6 +114,20 @@ public void removeLanguage(String language) } } + public TranslationResult getSingleTranslation(TranscriptionResult result, + String targetLanguage) + { + String translatedText = translationService.translate( + result.getAlternatives().iterator().next().getTranscription(), + result.getParticipant().getSourceLanguage(), + targetLanguage); + + return new TranslationResult( + result, + targetLanguage, + translatedText); + } + /** * Translates the received {@link TranscriptionResult} into required languages * and returns a list of {@link TranslationResult}s.