In my previous post I’ve shared a quick overview on the Azure Computer Vision API service and it’s implementation. If you didn’t read it yet, please do before proceeding to this reading!
With the basics and the CognitiveServices in place, let’s move forward and create a custom image field that uses this service to handle the image cropping, tagging and alt text description, all with AI.
I’ll be sharing the whole implementation in GitHub later and also a package plugin, but let’s get into the implementation details first.
Custom Image Field
The first step is to create the custom field, for doing that, go to the core DB and duplicate the /sitecore/system/Field types/Simple Types/Image field item. Let’s call it “AICroppedImage“.
Keep everything as it is except the assembly and class fields
AICroppedImage Class
For the implementation, we just decompiled the code from Sitecore.Kernel (Sitecore.Shell.Applications.ContentEditor.Image) and made all our needed customizations.
using Sitecore.Configuration;
using Sitecore.Data.Items;
using Sitecore.DependencyInjection;
using Sitecore.Diagnostics;
using Sitecore.Globalization;
using Sitecore.Resources.Media;
using Sitecore.Shell.Applications.ContentEditor;
using Sitecore.Web.UI.Sheer;
using System;
using System.IO;
using System.Text;
using System.Web;
using System.Web.UI;
using Microsoft.Extensions.DependencyInjection;
using System.Linq;
using Sitecore.Computer.Vision.CroppingImageField.Models.ImagesDetails;
using Sitecore.Computer.Vision.CroppingImageField.Services;
namespace Sitecore.Computer.Vision.CroppingImageField.Fields
{
public class AICroppedImage : Image
{
private readonly string ThumbnailsId = Settings.GetSetting("Sitecore.Computer.Vision.CroppingImageField.AICroppingField.ThumbnailsFolderId");
private readonly ICognitiveServices _cognitiveServices;
private readonly ICroppingService _croppingService;
public AICroppedImage(ICognitiveServices cognitiveServices, ICroppingService croppingService) : base()
{
_cognitiveServices = cognitiveServices;
_croppingService = croppingService;
}
public AICroppedImage() : base()
{
_cognitiveServices = ServiceLocator.ServiceProvider.GetService<ICognitiveServices>();
_croppingService = ServiceLocator.ServiceProvider.GetService<ICroppingService>();
}
protected override void DoRender(HtmlTextWriter output)
{
Assert.ArgumentNotNull((object)output, nameof(output));
Item mediaItem = this.GetMediaItem();
string src;
this.GetSrc(out src);
string str1 = " src=\"" + src + "\"";
string str2 = " id=\"" + this.ID + "_image\"";
string str3 = " alt=\"" + (mediaItem != null ? HttpUtility.HtmlEncode(mediaItem["Alt"]) : string.Empty) + "\"";
this.Attributes["placeholder"] = Translate.Text(this.Placeholder);
string str = this.Password ? " type=\"password\"" : (this.Hidden ? " type=\"hidden\"" : "");
this.SetWidthAndHeightStyle();
output.Write("<input" + this.ControlAttributes + str + ">");
this.RenderChildren(output);
output.Write("<div id=\"" + this.ID + "_pane\" class=\"scContentControlImagePane\">");
string clientEvent = Sitecore.Context.ClientPage.GetClientEvent(this.ID + ".Browse");
output.Write("<div class=\"scContentControlImageImage\" onclick=\"" + clientEvent + "\">");
output.Write("<iframe" + str2 + str1 + str3 + " frameborder=\"0\" marginwidth=\"0\" marginheight=\"0\" width=\"100%\" height=\"128\" " +
"allowtransparency=\"allowtransparency\"></iframe>");
output.Write("<div id=\"" + this.ID + "_thumbnails\">");
output.Write(GetThumbnails());
output.Write("</div>");
output.Write("</div>");
output.Write("<div>");
output.Write("<div id=\"" + this.ID + "_details\" class=\"scContentControlImageDetails\">");
string details = this.GetDetails();
output.Write(details);
output.Write("</div>");
output.Write("</div>");
}
protected override void DoChange(Message message)
{
Assert.ArgumentNotNull((object)message, nameof(message));
base.DoChange(message);
if (Sitecore.Context.ClientPage.Modified)
{
this.Update();
}
if (string.IsNullOrEmpty(this.Value))
{
this.ClearImage();
}
SheerResponse.SetReturnValue(true);
}
protected new void BrowseImage(ClientPipelineArgs args)
{
Assert.ArgumentNotNull((object)args, nameof(args));
base.BrowseImage(args);
if (Sitecore.Context.ClientPage.Modified)
{
this.Update();
}
}
protected new void ShowProperties(ClientPipelineArgs args)
{
Assert.ArgumentNotNull((object)args, nameof(args));
base.ShowProperties(args);
if (Sitecore.Context.ClientPage.Modified)
{
this.Update();
}
}
public override void HandleMessage(Message message)
{
Assert.ArgumentNotNull((object)message, nameof(message));
base.HandleMessage(message);
string name = message.Name;
if (name == "contentimage:clear")
{
this.ClearImage();
}
else if (name == "contentimage:refresh")
{
this.Update();
}
}
private void ClearImage()
{
if (this.Disabled)
{
return;
}
if (this.Value.Length > 0)
{
this.SetModified();
}
this.XmlValue = new XmlValue(string.Empty, "image");
this.Value = string.Empty;
this.Update();
}
protected new void Update()
{
string src;
this.GetSrc(out src);
SheerResponse.SetAttribute(this.ID + "_image", "src", src);
SheerResponse.SetInnerHtml(this.ID + "_thumbnails", this.GetThumbnails());
SheerResponse.SetInnerHtml(this.ID + "_details", this.GetDetails());
SheerResponse.Eval("scContent.startValidators()");
}
private string GetDetails()
{
var empty = string.Empty;
MediaItem mediaItem = this.GetMediaItem();
if (mediaItem != null)
{
var innerItem = mediaItem.InnerItem;
var stringBuilder = new StringBuilder();
var xmlValue = this.XmlValue;
stringBuilder.Append("<div>");
var item = innerItem["Dimensions"];
var str = HttpUtility.HtmlEncode(xmlValue.GetAttribute("width"));
var str1 = HttpUtility.HtmlEncode(xmlValue.GetAttribute("height"));
ImageDetails imageDetails;
using (var streamReader = new MemoryStream())
{
var mediaStrm = mediaItem.GetMediaStream();
mediaStrm.CopyTo(streamReader);
imageDetails = _cognitiveServices.AnalyzeImage(streamReader.ToArray());
}
if (!string.IsNullOrEmpty(str) || !string.IsNullOrEmpty(str1))
{
var objArray = new object[] { str, str1, item };
stringBuilder.Append(Translate.Text("Dimensions: {0} x {1} (Original: {2})", objArray));
}
else
{
var objArray1 = new object[] { item };
stringBuilder.Append(Translate.Text("Dimensions: {0}", objArray1));
}
stringBuilder.Append("</div>");
stringBuilder.Append("<div style=\"padding:2px 0px 0px 0px; text-align=left; \">");
var str2 = HttpUtility.HtmlEncode(innerItem["Alt"]);
var str3 = imageDetails.Description.Captions.FirstOrDefault()?.Text;
if (!string.IsNullOrEmpty(str3) && !string.IsNullOrEmpty(str2))
{
var objArray2 = new object[] { str3, str2 };
stringBuilder.Append(Translate.Text("AI Alternate Text: \"{0}\" (Default Alternate Text: \"{1}\")", objArray2));
}
else if (!string.IsNullOrEmpty(str3))
{
var objArray3 = new object[] { str3 };
stringBuilder.Append(Translate.Text("AI Alternate Text: \"{0}\"", objArray3));
}
else
{
var objArray4 = new object[] { str2 };
stringBuilder.Append(Translate.Text("Default Alternate Text: \"{0}\"", objArray4));
}
stringBuilder.Append("</br>");
var objArray5 = new object[] { str3 };
stringBuilder.Append(Translate.Text("Tags: \"{0}\"", string.Join(",", imageDetails.Description.Tags), objArray5));
stringBuilder.Append("</div>");
empty = stringBuilder.ToString();
}
if (empty.Length == 0)
{
empty = Translate.Text("This media item has no details.");
}
return empty;
}
private Item GetMediaItem()
{
var attribute = this.XmlValue.GetAttribute("mediaid");
if (attribute.Length <= 0)
{
return null;
}
Language language = Language.Parse(this.ItemLanguage);
return Sitecore.Client.ContentDatabase.GetItem(attribute, language);
}
private MediaItem GetSrc(out string src)
{
src = string.Empty;
MediaItem mediaItem = (MediaItem)this.GetMediaItem();
if (mediaItem == null)
{
return null;
}
var thumbnailOptions = MediaUrlOptions.GetThumbnailOptions(mediaItem);
int result;
if (!int.TryParse(mediaItem.InnerItem["Height"], out result))
{
result = 128;
}
thumbnailOptions.Height = Math.Min(128, result);
thumbnailOptions.MaxWidth = 640;
thumbnailOptions.UseDefaultIcon = true;
src = MediaManager.GetMediaUrl(mediaItem, thumbnailOptions);
return mediaItem;
}
private string GetThumbnails()
{
var html = new StringBuilder();
var src = string.Empty;
var mediaItem = this.GetSrc(out src);
if (mediaItem == null)
{
return string.Empty;
}
html.Append("<ul id=" + this.ID + "_frame\" style=\"display: -ms-flexbox;display: flex;-ms-flex-direction: row;flex-direction: row;-ms-flex-wrap: wrap;flex-wrap: wrap;\">");
var thumbnailFolderItem = Sitecore.Client.ContentDatabase.GetItem(new Sitecore.Data.ID(ThumbnailsId));
if (thumbnailFolderItem != null && thumbnailFolderItem.HasChildren)
{
foreach (Item item in thumbnailFolderItem.Children)
{
GetThumbnailHtml(item, html, mediaItem);
}
}
html.Append("</ul>");
return html.ToString();
}
private void GetThumbnailHtml(Item item, StringBuilder html, MediaItem mediaItem)
{
if (item.Fields["Size"]?.Value != null)
{
var values = item.Fields["Size"].Value.Split('x');
var width = values[0];
var height = values[1];
int w, h;
if (int.TryParse(width, out w) && Int32.TryParse(height, out h) && w > 0 && h > 0)
{
var imageSrc = _croppingService.GenerateThumbnailUrl(w, h, mediaItem);
html.Append(string.Format("<li id=\"Frame_{0}_{1}\" style=\"width: {2}px; height: {3}px; position: relative; overflow: hidden; display: inline-block;border: solid 3px #fff;margin: 5px 5px 5px 0;\">" +
"<img style=\"position: relative;position: absolute;left: 0;top: 0;margin: 0;display: block;width: auto; height: auto;min-width: 100%; min-height: 100%;max-height: none; max-width: none;\" " +
"src=\"{4}\"><img /><span style=\"position: absolute;" +
"top: 0;left: 0;padding: 2px 3px;background-color: #fff;opacity: 0.8;\">{5}</span></li>", this.ID, item.ID.ToShortID(), w, h, imageSrc, item.DisplayName));
}
}
}
}
}
We’re basically modifying the way Sitecore renders the field with some small customizations, basically to add the thumbnails generated by the Azure Cognitive service and also the Alt and Tags texts.
Ok, so that’s very much it, let’s deploy our code and see how it looks in the Sitecore Content Editor. The only thing you need to do next, is create a template and make use of the newly created “AI Cropped Image” field.
Et Voila! The image field is now rendering a few thumbnails that gives you an idea of the final results when rendering the image in the front-end. As you can see, it gives also some tags and a description (“Diego Maradona holding a ball”) used as alt text, everything coming from the Azure AI service, awesome!
Make the field rendered to work as an OOTB Sitecore image field
Next step, is to make sure we can still using the Sitecore helpers for rendering this field. For making this possible, we want to customize the Sitecore.Pipelines.RenderField.GetImageFieldValue processor. Same as before, we decompile the OOTB code from Sitecore.Kernel and we make our updates there. Then just patch the config like that:
<pipelines>
<renderField>
<processor patch:after="processor[@type='Sitecore.Pipelines.RenderField.GetImageFieldValue, Sitecore.Kernel']"
type="Foundation.Vision.Pipelines.RenderAICroppingImageField, Foundation.Vision">
</processor>
</renderField>
</pipelines>
Here, we just need to add the newly created field type (AI Cropped Image) as a valid image field type by overriding the IsImage() method.
using Sitecore.Diagnostics;
using Sitecore.Pipelines.RenderField;
namespace Sitecore.Computer.Vision.CroppingImageField.Pipelines
{
public class RenderAICroppingImageField : GetImageFieldValue
{
public override void Process(RenderFieldArgs args)
{
Assert.ArgumentNotNull((object)args, nameof(args));
if (!this.IsImage(args))
{
return;
}
var renderer = this.CreateRenderer();
this.ConfigureRenderer(args, renderer);
this.SetRenderFieldResult(renderer.Render(), args);
}
protected override bool IsImage(RenderFieldArgs args)
{
return args.FieldTypeKey == "AI Cropped Image";
}
}
}
Make it working with GlassMapper
Now, we can do some quick updates to GlassMapper as well so we can benefit from the glass helpers. Let’s add a custom field mapper, again after decompiling Glass.Mapper.Sc.DataMappers.SitecoreFieldImageMapper, we can just extend it to work in the same way with the newly introduced AI Cropping Image field.
using Glass.Mapper.Sc;
using Glass.Mapper.Sc.Configuration;
using Glass.Mapper.Sc.DataMappers;
using Sitecore.Data;
using Sitecore.Data.Fields;
using Sitecore.Data.Items;
using System;
using Sitecore.Computer.Vision.CroppingImageField.Fields;
namespace Sitecore.Computer.Vision.CroppingImageField.Mappers
{
public class AICroppedImageFieldMapper : AbstractSitecoreFieldMapper
{
public AICroppedImageFieldMapper(): base(typeof(AICroppedImage))
{
}
public override object GetField(Field field, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var img = new AICroppedImage();
var sitecoreImage = new AICroppedImageField(field);
SitecoreFieldImageMapper.MapToImage(img, sitecoreImage);
return img;
}
public override void SetField(Field field, object value, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var img = value as AICroppedImage;
if (field == null || img == null)
{
return;
}
var item = field.Item;
var sitecoreImage = new AICroppedImageField(field);
SitecoreFieldImageMapper.MapToField(sitecoreImage, img, item);
}
public override string SetFieldValue(object value, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
throw new NotImplementedException();
}
public override object GetFieldValue(string fieldValue, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var item = context.Service.Database.GetItem(new ID(fieldValue));
if (item == null)
{
return null;
}
var imageItem = new MediaItem(item);
var image = new AICroppedImage();
SitecoreFieldImageMapper.MapToImage(image, imageItem);
return image;
}
}
}
We need also to create our custom field that inherits from Glass.Mapper.Sc.Fields.Image
using Glass.Mapper.Sc.Fields;
namespace Sitecore.Computer.Vision.CroppingImageField.Mappers
{
public class AICroppedImage : Image
{
}
}
Last step is to add the mapper to the create resolver from the GlassMapperSCCustom.cs
public static class GlassMapperScCustom
{
public static IDependencyResolver CreateResolver(){
var config = new Glass.Mapper.Sc.Config();
var dependencyResolver = new DependencyResolver(config);
// add any changes to the standard resolver here
dependencyResolver.DataMapperFactory.First(() => new AICroppedImageFieldMapper());
dependencyResolver.Finalise();
return dependencyResolver;
}
}
Custom Caching
In order to reduce the calls to the service, an extra layer of caching has been implemented. This cache, as any other Sitecore cache gets flushed after a publishing and the size can be easily configured through it’s configuration.
...
<setting name="Sitecore.Computer.Vision.CroppingImageField.AICroppingField.CognitiveServices.CacheSize" value="100MB" />
</settings>
<events>
<event name="publish:end:remote">
<handler type="Sitecore.Computer.Vision.CroppingImageField.Caching.CacheManager, Sitecore.Computer.Vision.CroppingImageField" method="ClearCache" />
</event>
<event name="customCache:rebuild:remote">
<handler type="Sitecore.Computer.Vision.CroppingImageField.Caching.CacheManager, Sitecore.Computer.Vision.CroppingImageField" method="ClearCache" />
</event>
</events>
...
In my next post, I’ll be sharing the front end implementation, the full media request flow and the customizations needed to make it working in your site. Stay tuned!
Miguel Minoldo 
Pingback: Sitecore smart image cropping, tags and alt text with AI: Azure Computer Vision – Part III. | Miguel Minoldo
Pingback: Sitecore smart image cropping, tags and alt text with AI: Azure Computer Vision – Part I. | Miguel Minoldo