In my previous post I’ve shared the custom image field implementation that makes use of the Azure Computer Vision service in order to crop and generate the thumbnails using AI. Please before proceed with this reading, make sure you already went through the previous posts: Part I and Part II.
Now, I’ll be sharing the last, but not least part of this topic, how to make it working in the front-end side, the media request flow and so on.
Image request flow
So, the request flow is described in the following graph, basically follows the normal Sitecore flow but with the introduction of the Azure Computer Vision and Image Sharp to generate the proper cropping version of the image.
AICroppingProcessor
This custom processor will be overriding the Sitecore OOTB ThumbnailProcessor. It’s basically a copy from the original code with a customization to check the “SmartCropping” parameter from the image request.
using Sitecore.Diagnostics;
using Sitecore.Resources.Media;
using System;
using Microsoft.Extensions.DependencyInjection;
using System.IO;
using System.Linq;
using Sitecore.Computer.Vision.CroppingImageField.Services;
using Sitecore.DependencyInjection;
namespace Sitecore.Computer.Vision.CroppingImageField.Processors
{
public class AICroppingProcessor
{
private static readonly string[] AllowedExtensions = { "bmp", "jpeg", "jpg", "png", "gif" };
private readonly ICroppingService _croppingService;
public AICroppingProcessor(ICroppingService croppingService)
{
_croppingService = croppingService;
}
public AICroppingProcessor()
{
_croppingService = ServiceLocator.ServiceProvider.GetService<ICroppingService>();
}
public void Process(GetMediaStreamPipelineArgs args)
{
Assert.ArgumentNotNull(args, "args");
var outputStream = args.OutputStream;
if (outputStream == null)
{
return;
}
if (!AllowedExtensions.Any(i => i.Equals(args.MediaData.Extension, StringComparison.InvariantCultureIgnoreCase)))
{
return;
}
var smartCrop = args.Options.CustomOptions[Constants.QueryStringKeys.SmartCropping];
if (!string.IsNullOrEmpty(smartCrop) && bool.Parse(smartCrop))
{
Stream outputStrm;
outputStrm = Stream.Synchronized(_croppingService.GetCroppedImage(args.Options.Width, args.Options.Height, outputStream.MediaItem));
args.OutputStream = new MediaStream(outputStrm, args.MediaData.Extension, outputStream.MediaItem);
}
else if (args.Options.Thumbnail)
{
var transformationOptions = args.Options.GetTransformationOptions();
var thumbnailStream = args.MediaData.GetThumbnailStream(transformationOptions);
if (thumbnailStream != null)
{
args.OutputStream = thumbnailStream;
}
}
}
}
}
We need also to customize the MediaRequest to also take the “SmartCropping” parameter into account:
using Sitecore.Configuration;
using Sitecore.Diagnostics;
using Sitecore.Resources.Media;
using System.Web;
namespace Sitecore.Computer.Vision.CroppingImageField.Requests
{
using System.Collections.Specialized;
public class AICroppingMediaRequest : MediaRequest
{
private HttpRequest _innerRequest;
private MediaUrlOptions _mediaQueryString;
private MediaUri _mediaUri;
private MediaOptions _options;
protected override MediaOptions GetOptions()
{
var queryString = this.InnerRequest.QueryString;
if (queryString == null || queryString.Count == 0)
{
_options = new MediaOptions();
}
else
{
SetMediaOptionsFromMediaQueryString(queryString);
if (!string.IsNullOrEmpty(queryString.Get(Constants.QueryStringKeys.SmartCropping)))
{
SetCustomOptionsFromQueryString(queryString);
}
}
if (!this.IsRawUrlSafe)
{
if (Settings.Media.RequestProtection.LoggingEnabled)
{
string urlReferrer = this.GetUrlReferrer();
Log.SingleError(string.Format("MediaRequestProtection: An invalid/missing hash value was encountered. " +
"The expected hash value: {0}. Media URL: {1}, Referring URL: {2}",
HashingUtils.GetAssetUrlHash(this.InnerRequest.RawUrl), this.InnerRequest.RawUrl,
string.IsNullOrEmpty(urlReferrer) ? "(empty)" : urlReferrer), this);
}
_options = new MediaOptions();
}
return _options;
}
private void SetCustomOptionsFromQueryString(NameValueCollection queryString)
{
this.ProcessCustomParameters(_options);
if (!string.IsNullOrEmpty(queryString.Get(Constants.QueryStringKeys.SmartCropping))
&& !_options.CustomOptions.ContainsKey(Constants.QueryStringKeys.SmartCropping)
&& !string.IsNullOrEmpty(queryString.Get(Constants.QueryStringKeys.SmartCropping)))
{
_options.CustomOptions.Add(Constants.QueryStringKeys.SmartCropping, queryString.Get(Constants.QueryStringKeys.SmartCropping));
}
}
private void SetMediaOptionsFromMediaQueryString(NameValueCollection queryString)
{
MediaUrlOptions mediaQueryString = this.GetMediaQueryString();
_options = new MediaOptions()
{
AllowStretch = mediaQueryString.AllowStretch,
BackgroundColor = mediaQueryString.BackgroundColor,
IgnoreAspectRatio = mediaQueryString.IgnoreAspectRatio,
Scale = mediaQueryString.Scale,
Width = mediaQueryString.Width,
Height = mediaQueryString.Height,
MaxWidth = mediaQueryString.MaxWidth,
MaxHeight = mediaQueryString.MaxHeight,
Thumbnail = mediaQueryString.Thumbnail,
UseDefaultIcon = mediaQueryString.UseDefaultIcon
};
if (mediaQueryString.DisableMediaCache)
{
_options.UseMediaCache = false;
}
foreach (string allKey in queryString.AllKeys)
{
if (allKey != null && queryString[allKey] != null)
{
_options.CustomOptions[allKey] = queryString[allKey];
}
}
}
public override MediaRequest Clone()
{
Assert.IsTrue((base.GetType() == typeof(AICroppingMediaRequest)), "The Clone() method must be overridden to support prototyping.");
return new AICroppingMediaRequest
{
_innerRequest = this._innerRequest,
_mediaUri = this._mediaUri,
_options = this._options,
_mediaQueryString = this._mediaQueryString
};
}
}
}
This code is very straightforward, it will basically check if the “SmartCropping=true” parameter exists in the media request, and then executes the custom code to crop the image.
The “Get Thumbnails” method limitations
As we can see in the official documentation, there are some limitations on the thumbnail generator method.
- Image file size must be less than 4MB.
- Image dimensions should be greater than 50 x 50.
- Width of the thumbnail must be between 1 and 1024.
- Height of the thumbnail must be between 1 and 1024.
The most important one is that the width and height cannot exceed the 1024px, this is problematic as sometimes we need to crop on a bigger ratio.
So, in order to make it more flexible, I’m doing the cropping using the Graphics library but getting the focus point coordinates from the “Get Area Of Interest” API method:
using Sitecore.Data.Items;
using Microsoft.Extensions.DependencyInjection;
using System.IO;
using Sitecore.DependencyInjection;
using Sitecore.Resources.Media;
using System.Drawing;
using System.Drawing.Imaging;
using System.Drawing.Drawing2D;
namespace Sitecore.Computer.Vision.CroppingImageField.Services
{
public class CroppingService : ICroppingService
{
private readonly ICognitiveServices _cognitiveServices;
public CroppingService(ICognitiveServices cognitiveServices)
{
_cognitiveServices = cognitiveServices;
}
public CroppingService()
{
_cognitiveServices = ServiceLocator.ServiceProvider.GetService<ICognitiveServices>();
}
public Stream GetCroppedImage(int width, int height, MediaItem mediaItem)
{
using (var streamReader = new MemoryStream())
{
var mediaStrm = mediaItem.GetMediaStream();
mediaStrm.CopyTo(streamReader);
mediaStrm.Position = 0;
var img = Image.FromStream(mediaStrm);
// The cropping size shouldn't be higher than the original image
if (width > img.Width || height > img.Height)
{
Sitecore.Diagnostics.Log.Warn($"Media file is smaller than the requested crop size. " +
$"This can result on a low quality result. Please upload a proper image: " +
$"Min Height:{height}, Min Width:{width}. File: {mediaItem.DisplayName}, Path{mediaItem.MediaPath}", this);
}
// if the cropping size exceeds the cognitive services limits, get the focus point and crop
if (width > 1025 || height > 1024)
{
var area = _cognitiveServices.GetAreaOfImportance(streamReader.ToArray());
var cropImage = CropImage(img, area.areaOfInterest.X, area.areaOfInterest.Y, width, height);
return cropImage;
}
var thumbnailResult = _cognitiveServices.GetThumbnail(streamReader.ToArray(), width, height);
return new MemoryStream(thumbnailResult);
}
}
public string GenerateThumbnailUrl(int width, int height, MediaItem mediaItem)
{
var streamReader = MediaManager.GetMedia(mediaItem).GetStream();
{
using (var memStream = new MemoryStream())
{
streamReader.Stream.CopyTo(memStream);
var thumbnail = _cognitiveServices.GetThumbnail(memStream.ToArray(), width, height);
var imreBase64Data = System.Convert.ToBase64String(thumbnail);
return $"data:image/png;base64,{imreBase64Data}";
}
}
}
private Stream CropImage(Image source, int x, int y, int width, int height)
{
var bmp = new Bitmap(width, height);
var outputStrm = new MemoryStream();
using (var gr = Graphics.FromImage(bmp))
{
gr.InterpolationMode = InterpolationMode.HighQualityBicubic;
using (var wrapMode = new ImageAttributes())
{
wrapMode.SetWrapMode(WrapMode.TileFlipXY);
gr.DrawImage(source, new Rectangle(0, 0, bmp.Width, bmp.Height), x, y, width, height, GraphicsUnit.Pixel, wrapMode);
}
}
bmp.Save(outputStrm, source.RawFormat);
return outputStrm;
}
}
}
Let’s see this in action!
After picking your picture in the AI Cropping Image field, it gets already cropped and you can see the different thumbnails. You can choose or change the thumbnails by updating the child items here: /sitecore/system/Settings/Foundation/Vision/Thumbnails.
Also note that you get an auto generated Alt text “Diego Maradona holding a ball” and a list of tags.
The results
This is how the different cropped images will look like in the front end. Depending on your front end implementation, you will define different cropping sizes per breakpoints.
In this following implementation, I’m setting the image as a background and using the option to render the image URL as follows:
<div class="heroBanner__backgroundWrapper">
<div v-animate-on-inview="{class: 'animateScaleOut', delay: 10}"
v-animate-on-scroll="{class: 'animateOverlay'}"
class="heroBanner__background @Model.HeroClass" role="img" aria-label="@Model.GlassModel.ProductHeroImage.Alt"
v-background="{
'0': '@Html.Sitecore().AICroppingImageField("AI Image", Model.GlassModel.Item, new AdvancedImageParameters {Width = 600, Height = 600, OnlyUrl = true})',
'360': '@Html.Sitecore().AICroppingImageField("AI Image", Model.GlassModel.Item, new AdvancedImageParameters {Width = 900, Height = 900, OnlyUrl = true})',
'720': '@Html.Sitecore().AICroppingImageField("AI Image", Model.GlassModel.Item, new AdvancedImageParameters {Width = 1667, Height = 750, OnlyUrl = true})',
'1280': '@Html.Sitecore().AICroppingImageField("AI Image", Model.GlassModel.Item, new AdvancedImageParameters {Width = 2000, Height = 900, OnlyUrl = true})'
}">
</div>
Sitecore Media Cache
As I mentioned before, the cropped images are also stored in the media cache, as we can confirm by checking the media cache folder
Other usages and helpers
Sitecore HTML helper
You can use the @Sitecore.Html helper to render an image tag, as usual, or to generate just the URL of the image (src).
Code
@Html.Sitecore().AICroppingImageField("AI Image", Model.Item, new AdvancedImageParameters { Width = 600, Height = 600, AutoAltText = true })
Result
<img alt="a close up of a person wearing glasses"
src="https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=600&h=600&smartCropping=true&hash=C2E215FE2CF74D4C8142E35619ABB8DE">
Note: Have a look at the AdvancedImageParameters:
- OnlyUrl: If true it will just render the image URL (for being used as src in the img tag).
- AutoAltText: If true, the alt text will be replaced by the one generated from Azure IA.
- Width and Height: int values, to specify the cropping size.
- Widths and Sizes: If set, it will generate a srcset image with for the different breakpoints.
- SizesTag and SrcSetTag: Those are mandatories if when using the previous settings.
Code
@Html.Sitecore().AICroppingImageField("AI Image", Model.Item, new
AdvancedImageParameters {Widths = "170,233,340,466", Sizes = "50vw,(min-width:
999px) 25vw,(min-width: 1200px) 15vw", SizesTag = "data-sizes", SrcSetTag = "data-
srcset", AutoAltText = true })
Result
<img alt="a close up of a person wearing glasses" data-sizes="50vw,(min-width:
999px) 25vw,(min-width: 1200px) 15vw" data-
srcset="https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=170&hash=1D04C1F551E9606AB2EEB3C712255651
170w,https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=233&hash=DD2844D340246D3CF8AEBB63CE4E9397
233w,https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=340&hash=3B773ACB5136214979A0009E24F25F02
340w,https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=466&hash=424F7615FBECFED21F48DA0AE1FE7A5B 466w"
src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==">
GlassMapper extension
At last, an extension method has been added in order to get the media URL from the image field.
Code
<img src="@Model.AiImage.GetImageUrl(600, 600)" />
Result
<img src="https://vision.test.cm/-/media/project/vision/homepage/iatestimage.png?
w=600&h=600&smartCropping=true&hash=C2E215FE2CF74D4C8142E35619ABB8DE">
Sitecore Package and code
Please find the whole implementation in my GitHub repo, also feel free to contribute 🙂
You can also download the Sitecore package from here. Note: It has been tested on Sitecore 8.2, 9.x and 10.x.
You can also get the Docker asset image from Docker Hub!
docker pull miguelminoldo/sitecore.computer.vision
That’s it! I hope you find it interesting and useful! Any feedback is always welcome!
Miguel Minoldo 
Pingback: Sitecore smart image cropping, tags and alt text with AI: Azure Computer Vision – Part II. | Miguel Minoldo