Skip to content
This repository has been archived by the owner on Feb 10, 2022. It is now read-only.

Commit

Permalink
Text Analytics updates (#125)
Browse files Browse the repository at this point in the history 
* Text Analytics updates

* fix type
  • Loading branch information
Albert Davletov authored Feb 9, 2021
1 parent 810ae85 commit 0d5bccd
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 89 deletions.
2 changes: 1 addition & 1 deletion Kiosk/IntelligentKioskSample.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -882,7 +882,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.TextAnalytics">
<Version>5.1.0-beta.2</Version>
<Version>5.1.0-beta.3</Version>
</PackageReference>
<PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.ComputerVision">
<Version>6.0.0</Version>
Expand Down
2 changes: 1 addition & 1 deletion Kiosk/ServiceHelpers/ServiceHelpers.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.TextAnalytics">
<Version>5.1.0-beta.2</Version>
<Version>5.1.0-beta.3</Version>
</PackageReference>
<PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.ComputerVision">
<Version>6.0.0</Version>
Expand Down
56 changes: 39 additions & 17 deletions Kiosk/ServiceHelpers/TextAnalyticsHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,27 @@
using Azure;
using Azure.AI.TextAnalytics;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;

namespace ServiceHelpers
{
public static class TextAnalyticsHelper
{
// NOTE 10/19/2020: Text Analytics API v3 is not available in the following regions: China North 2, China East.
// https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/migration-guide?tabs=sentiment-analysis
public static readonly string[] NotAvailableAzureRegions = new string[] { "chinanorth2", "chinaeast" };
// NOTE 12/17/2020: Text Analytics API v3 language support
// See details: https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/language-support
public static readonly string DefaultLanguageCode = "en";
public static readonly string[] SentimentAnalysisSupportedLanguages = { "zh", "zh-hans", "zh-hant", "en", "fr", "de", "hi", "it", "ja", "ko", "no", "pt", "pt-BR", "pt-PT", "es", "tr" };
public static readonly string[] OpinionMiningSupportedLanguages = { "en" };
public static readonly string[] KeyPhraseExtractionSupportedLanguages = { "da", "nl", "en", "fi", "fr", "de", "it", "ja", "ko", "no", "nb", "pl", "pt", "pt-BR", "pt-PT", "ru", "es", "sv" };
public static readonly string[] NamedEntitySupportedLanguages = { "ar", "zh", "zh-hans", "zh-hant", "cs", "da", "nl", "en", "fi", "fr", "de", "he", "hu", "it", "ja", "ko", "no", "nb", "pl", "pt", "pt-BR", "pt-PT", "ru", "es", "sv", "tr" };
public static readonly string[] EntityLinkingSupportedLanguages = { "en", "es" };
public static readonly Uri LanguageSupportUri = new Uri("https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/language-support");
public static readonly Dictionary<string, string> LanguageCodeMap = new Dictionary<string, string>()
{
{ "zh_chs", "zh-hans" },
{ "zh_cht", "zh-hant" }
};

// Note: Data limits
// See details: https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/concepts/data-limits?tabs=version-3#data-limits
Expand Down Expand Up @@ -97,41 +109,51 @@ private static void InitializeTextAnalyticsService()
client = credentials != null && endpoint != null ? new TextAnalyticsClient(endpoint, credentials) : null;
}

public static async Task<DocumentSentiment> AnalyzeSentimentAsync(string input, string language = "en", AdditionalSentimentAnalyses sentimentAnalyses = AdditionalSentimentAnalyses.None)
public static async Task<DetectedLanguage> DetectLanguageAsync(string input)
{
return await client.DetectLanguageAsync(input);
}

public static async Task<DocumentSentiment> AnalyzeSentimentAsync(string input, string language = null, bool includeOpinionMining = false)
{
var options = new AnalyzeSentimentOptions() { AdditionalSentimentAnalyses = sentimentAnalyses };
var options = new AnalyzeSentimentOptions() { IncludeOpinionMining = includeOpinionMining };
return await client.AnalyzeSentimentAsync(input, language, options);
}

public static async Task<AnalyzeSentimentResultCollection> AnalyzeSentimentAsync(string[] input, string language = "en", AdditionalSentimentAnalyses sentimentAnalyses = AdditionalSentimentAnalyses.None)
public static async Task<AnalyzeSentimentResultCollection> AnalyzeSentimentAsync(string[] input, string language = null, bool includeOpinionMining = false)
{
var options = new AnalyzeSentimentOptions() { AdditionalSentimentAnalyses = sentimentAnalyses };
var options = new AnalyzeSentimentOptions() { IncludeOpinionMining = includeOpinionMining };
return await client.AnalyzeSentimentBatchAsync(input, language, options);
}

public static async Task<DetectedLanguage> DetectLanguageAsync(string input)
public static async Task<KeyPhraseCollection> ExtractKeyPhrasesAsync(string input, string language = null)
{
return await client.DetectLanguageAsync(input);
return await client.ExtractKeyPhrasesAsync(input, language);
}

public static async Task<CategorizedEntityCollection> RecognizeEntitiesAsync(string input)
public static async Task<ExtractKeyPhrasesResultCollection> ExtractKeyPhrasesAsync(string[] input, string language = null)
{
return await client.RecognizeEntitiesAsync(input);
return await client.ExtractKeyPhrasesBatchAsync(input, language);
}

public static async Task<LinkedEntityCollection> RecognizeLinkedEntitiesAsync(string input)
public static async Task<CategorizedEntityCollection> RecognizeEntitiesAsync(string input, string language = null)
{
return await client.RecognizeLinkedEntitiesAsync(input);
return await client.RecognizeEntitiesAsync(input, language);
}

public static async Task<KeyPhraseCollection> ExtractKeyPhrasesAsync(string input, string language = "en")
public static async Task<LinkedEntityCollection> RecognizeLinkedEntitiesAsync(string input, string language = null)
{
return await client.ExtractKeyPhrasesAsync(input, language);
return await client.RecognizeLinkedEntitiesAsync(input, language);
}

public static async Task<ExtractKeyPhrasesResultCollection> ExtractKeyPhrasesAsync(string[] input, string language = "en")
public static string GetLanguageCode(DetectedLanguage detectedLanguage)
{
return await client.ExtractKeyPhrasesBatchAsync(input, language);
if (LanguageCodeMap.ContainsKey(detectedLanguage.Iso6391Name))
{
return LanguageCodeMap[detectedLanguage.Iso6391Name];
}

return !string.IsNullOrEmpty(detectedLanguage.Iso6391Name) ? detectedLanguage.Iso6391Name : DefaultLanguageCode;
}
}
}
11 changes: 1 addition & 10 deletions Kiosk/Views/BingNewsAnalytics.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,7 @@ protected override async void OnNavigatedTo(NavigationEventArgs e)
}
else
{
bool isNotAvailableRegion = TextAnalyticsHelper.NotAvailableAzureRegions.Any(r => SettingsHelper.Instance.TextAnalyticsApiKeyEndpoint.Contains(r, StringComparison.OrdinalIgnoreCase));
if (isNotAvailableRegion)
{
this.page.IsEnabled = false;
await new MessageDialog("Text Analytics API v3 is not available in the following regions: China North 2, China East. Please change your Text Analytics key and region in the Settings page to a supported region.", "API key not supported").ShowAsync();
}
else
{
this.page.IsEnabled = true;
}
this.page.IsEnabled = true;
}

base.OnNavigatedTo(e);
Expand Down
130 changes: 70 additions & 60 deletions Kiosk/Views/TextAnalyticsExplorer/TextAnalyticsExplorer.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,18 @@ namespace IntelligentKioskSample.Views.TextAnalyticsExplorer
ExperienceType = ExperienceType.Guided | ExperienceType.Business,
TechnologiesUsed = TechnologyType.TextAnalytics,
TechnologyArea = TechnologyAreaType.Language,
DateAdded = "2020/09/17")]
DateAdded = "2020/09/17",
DateUpdated = "2021/02/04",
UpdatedDescription = "Now supporting more languages")]
public sealed partial class TextAnalyticsExplorer : Page
{
private static readonly Color PositiveColor = Color.FromArgb(255, 137, 196, 2); // #89c402
private static readonly Color NeutralColor = Color.FromArgb(255, 0, 120, 212); // #0078d4
private static readonly Color NegativeColor = Color.FromArgb(255, 165, 20, 25); // #a51419

private const string NotFound = "Not found";
private const string LanguageNotSupported = "Not supported in this language";

public ObservableCollection<MinedOpinion> OpinionMiningCollection { get; set; } = new ObservableCollection<MinedOpinion>();
public ObservableCollection<SampleText> SampleTextList { get; set; } = new ObservableCollection<SampleText>();

Expand All @@ -81,21 +86,12 @@ protected override async void OnNavigatedTo(NavigationEventArgs e)
}
else
{
bool isNotAvailableRegion = TextAnalyticsHelper.NotAvailableAzureRegions.Any(r => SettingsHelper.Instance.TextAnalyticsApiKeyEndpoint.Contains(r, StringComparison.OrdinalIgnoreCase));
if (isNotAvailableRegion)
{
this.mainPage.IsEnabled = false;
await new MessageDialog("Text Analytics API v3 is not available in the following regions: China North 2, China East. Please change your Text Analytics key and region in the Settings page to a supported region.", "API key not supported").ShowAsync();
}
else
this.mainPage.IsEnabled = true;
SampleTextList.AddRange(TextAnalyticsDataLoader.GetTextSamples());
if (SampleTextList.Any())
{
this.mainPage.IsEnabled = true;
SampleTextList.AddRange(TextAnalyticsDataLoader.GetTextSamples());
if (SampleTextList.Any())
{
this.sampleTextComboBox.SelectedIndex = 0;
await AnalyzeTextAsync();
}
this.sampleTextComboBox.SelectedIndex = 0;
await AnalyzeTextAsync();
}
}

Expand Down Expand Up @@ -129,58 +125,71 @@ private async Task AnalyzeTextAsync()
this.progressControl.IsActive = true;
DisplayProcessingUI();

// detect language
string input = this.inputText.Text;
var detectedLanguageTask = TextAnalyticsHelper.DetectLanguageAsync(input);
var detectedKeyPhrasesTask = TextAnalyticsHelper.ExtractKeyPhrasesAsync(input);
var documentSentimentTask = TextAnalyticsHelper.AnalyzeSentimentAsync(input, sentimentAnalyses: AdditionalSentimentAnalyses.OpinionMining);
var namedEntitiesResponseTask = TextAnalyticsHelper.RecognizeEntitiesAsync(input);
var linkedEntitiesResponseTask = TextAnalyticsHelper.RecognizeLinkedEntitiesAsync(input);

await Task.WhenAll(detectedLanguageTask, detectedKeyPhrasesTask, documentSentimentTask, namedEntitiesResponseTask, linkedEntitiesResponseTask);
var detectedLanguage = detectedLanguageTask.Result;
var detectedKeyPhrases = detectedKeyPhrasesTask.Result;
var documentSentiment = documentSentimentTask.Result;
var namedEntitiesResponse = namedEntitiesResponseTask.Result;
var linkedEntitiesResponse = linkedEntitiesResponseTask.Result;

// detected language and key phrases
this.detectedLangTextBlock.Text = !string.IsNullOrEmpty(detectedLanguage.Name) ? $"{detectedLanguage.Name} (confidence: {(int)(detectedLanguage.ConfidenceScore * 100)}%)" : "Not found";
this.detectedKeyPhrasesTextBlock.Text = detectedKeyPhrases.Any() ? string.Join(", ", detectedKeyPhrases) : "Not found";

// document sentiment
CreateSentimentChart(documentSentiment);

// mined opinions
OpinionMiningCollection.Clear();
var minedOpinions = documentSentiment?.Sentences.SelectMany(s => s.MinedOpinions);
if (minedOpinions != null && minedOpinions.Any())
{
var minedOpinionList = minedOpinions.Select(om => new MinedOpinion()
{
Aspect = om.Aspect.Text,
Opinions = string.Join(", ", om.Opinions.Select(o => $"{o.Text} ({o.Sentiment.ToString("G")})"))
});
OpinionMiningCollection.AddRange(minedOpinionList);
}
DetectedLanguage detectedLanguage = await TextAnalyticsHelper.DetectLanguageAsync(input);
string languageCode = TextAnalyticsHelper.GetLanguageCode(detectedLanguage);

// entities
if (namedEntitiesResponse.Any())
{
this.namesEntitiesGridView.ItemsSource = namedEntitiesResponse.Select(x => new { x.Text, Category = $"[{x.Category}]" });
}
else
{
this.namesEntitiesGridView.ItemsSource = new[] { new { Text = "No entities" } };
}
// check supported languages
bool isOpinionMiningSupported = TextAnalyticsHelper.OpinionMiningSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isSentimentSupported = TextAnalyticsHelper.SentimentAnalysisSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isKeyPhraseSupported = TextAnalyticsHelper.KeyPhraseExtractionSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isNamedEntitySupported = TextAnalyticsHelper.NamedEntitySupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isEntityLinkingSupported = TextAnalyticsHelper.EntityLinkingSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));

// sentiment analysis, key phrase extraction, named entity recognition and entity linking
Task<DocumentSentiment> documentSentimentTask = isSentimentSupported ? TextAnalyticsHelper.AnalyzeSentimentAsync(input, languageCode, isOpinionMiningSupported) : Task.FromResult<DocumentSentiment>(null);
Task<KeyPhraseCollection> detectedKeyPhrasesTask = isKeyPhraseSupported ? TextAnalyticsHelper.ExtractKeyPhrasesAsync(input, languageCode) : Task.FromResult<KeyPhraseCollection>(null);
Task<CategorizedEntityCollection> namedEntitiesResponseTask = isNamedEntitySupported ? TextAnalyticsHelper.RecognizeEntitiesAsync(input, languageCode) : Task.FromResult<CategorizedEntityCollection>(null);
Task<LinkedEntityCollection> linkedEntitiesResponseTask = isEntityLinkingSupported ? TextAnalyticsHelper.RecognizeLinkedEntitiesAsync(input, languageCode) : Task.FromResult<LinkedEntityCollection>(null);

await Task.WhenAll(documentSentimentTask, detectedKeyPhrasesTask, namedEntitiesResponseTask, linkedEntitiesResponseTask);

DocumentSentiment documentSentiment = documentSentimentTask.Result;
KeyPhraseCollection detectedKeyPhrases = detectedKeyPhrasesTask.Result;
CategorizedEntityCollection namedEntitiesResponse = namedEntitiesResponseTask.Result;
LinkedEntityCollection linkedEntitiesResponse = linkedEntitiesResponseTask.Result;

// display results
this.detectedLangTextBlock.Text = !string.IsNullOrEmpty(detectedLanguage.Name) ? $"{detectedLanguage.Name} (confidence: {(int)(detectedLanguage.ConfidenceScore * 100)}%)" : NotFound;

// linked entities
if (linkedEntitiesResponse.Any())
this.detectedKeyPhrasesTextBlock.Text = detectedKeyPhrases != null && detectedKeyPhrases.Any()
? string.Join(", ", detectedKeyPhrases)
: isKeyPhraseSupported ? NotFound : LanguageNotSupported;

this.namesEntitiesGridView.ItemsSource = namedEntitiesResponse != null && namedEntitiesResponse.Any()
? namedEntitiesResponse.Select(x => new { x.Text, Category = $"[{x.Category}]" })
: new[] { new { Text = isNamedEntitySupported ? "No entities" : LanguageNotSupported, Category = "" } };

this.linkedEntitiesGridView.ItemsSource = linkedEntitiesResponse != null && linkedEntitiesResponse.Any()
? linkedEntitiesResponse.Select(x => new { Name = $"{x.Name} ({x.DataSource})", x.Url })
: new[] {
isEntityLinkingSupported
? new { Name = "No linked entities", Url = new Uri("about:blank") }
: new { Name = LanguageNotSupported, Url = TextAnalyticsHelper.LanguageSupportUri }
};

if (isSentimentSupported)
{
this.linkedEntitiesGridView.ItemsSource = linkedEntitiesResponse.Select(x => new { Name = $"{x.Name} ({x.DataSource})", x.Url });
CreateSentimentChart(documentSentiment);

// mined opinions
OpinionMiningCollection.Clear();
var minedOpinions = documentSentiment?.Sentences.SelectMany(s => s.MinedOpinions);
if (minedOpinions != null && minedOpinions.Any())
{
var minedOpinionList = minedOpinions.Select(om => new MinedOpinion()
{
Aspect = om.Aspect.Text,
Opinions = string.Join(", ", om.Opinions.Select(o => $"{o.Text} ({o.Sentiment.ToString("G")})"))
});
OpinionMiningCollection.AddRange(minedOpinionList);
}
}
else
{
this.linkedEntitiesGridView.ItemsSource = new[] { new { Name = "No linked entities" } };
this.sentimentTextBlock.Text = LanguageNotSupported;
this.sentimentChart.Visibility = Visibility.Collapsed;
}

// prepare json result
Expand Down Expand Up @@ -239,6 +248,7 @@ private void DisplayProcessingUI()
this.detectedKeyPhrasesTextBlock.Text = label;
this.sentimentChart.Visibility = Visibility.Collapsed;
this.sentimentTextBlock.Text = label;
this.OpinionMiningCollection.Clear();
this.namesEntitiesGridView.ItemsSource = new[] { new { Text = label } };
this.linkedEntitiesGridView.ItemsSource = new[] { new { Name = label } };
}
Expand Down

0 comments on commit 0d5bccd

Please sign in to comment.