Computer Vision APIs (2024-02-01)
The Computer Vision API provides state-of-the-art algorithms to process images and return information. For example, it can be used to determine if an image contains mature content, or it can be used to find all the people in an image. It also has other features like categorizing the content of images, and describing an image with complete English sentences.
Analyze
Analyze the input image. The request either contains image stream with any content type ['image/*', 'application/octet-stream'], or a JSON payload which includes an url property to be used to retrieve the image stream.
Select the testing console in the region where you created your resource:
Open API testing consoleRequest URL
Request parameters
(optional)
array
The visual features requested: tags, objects, caption, denseCaptions, read, smartCrops, people. At least one visual feature must be specified.
(optional)
string
Optional parameter to specify the version of the AI model, or "latest" to use the latest available model. Defaults to "latest".
(optional)
string
The desired language for output generation. If this parameter is not specified, the default value is "en". See https://aka.ms/cv-languages for a list of supported languages.
(optional)
array
A list of aspect ratios to use for smartCrops feature. Aspect ratios are calculated by dividing the target crop width by the height. Supported values are between 0.75 and 1.8 (inclusive). Multiple values should be comma-separated. If this parameter is not specified, the service will return one crop suggestion with an aspect ratio it sees fit between 0.5 and 2.0 (inclusive).
(optional)
boolean
Boolean flag for enabling gender-neutral captioning for caption and denseCaptions features. If this parameter is not specified, the default value is "false".
Request headers
string
Media type of the body sent to the API.
string
Subscription key which provides access to this API. Found in your Cognitive Services accounts.
Request body
A JSON document with a URL pointing to the image that is to be analyzed.
{
"url": "string"
}{
"description": "A JSON document with a URL pointing to the image that is to be analyzed.",
"required": [
"url"
],
"type": "object",
"properties": {
"url": {
"description": "Publicly reachable URL of an image.",
"type": "string"
}
}
}Response 200
Success
{
"modelVersion": "string",
"captionResult": {
"text": "string",
"confidence": 0.0
},
"denseCaptionsResult": {
"values": [
{
"text": "string",
"confidence": 0.0,
"boundingBox": {
"x": 0,
"y": 0,
"w": 0,
"h": 0
}
}
]
},
"metadata": {
"width": 0,
"height": 0
},
"tagsResult": {
"values": [
{
"name": "string",
"confidence": 0.0
}
]
},
"objectsResult": {
"values": [
{
"id": "string",
"boundingBox": {
"x": 0,
"y": 0,
"w": 0,
"h": 0
},
"tags": [
{
"name": "string",
"confidence": 0.0
}
]
}
]
},
"readResult": {
"blocks": [
{
"lines": [
{
"text": "string",
"boundingPolygon": [
{
"x": 0,
"y": 0
}
],
"words": [
{
"text": "string",
"boundingPolygon": [
{
"x": 0,
"y": 0
}
],
"confidence": 0.0
}
]
}
]
}
]
},
"smartCropsResult": {
"values": [
{
"aspectRatio": 0.0,
"boundingBox": {
"x": 0,
"y": 0,
"w": 0,
"h": 0
}
}
]
},
"peopleResult": {
"values": [
{
"boundingBox": {
"x": 0,
"y": 0,
"w": 0,
"h": 0
},
"confidence": 0.0
}
]
}
}{
"description": "Describe the combined results of different types of image analysis.",
"required": [
"metadata",
"modelVersion"
],
"type": "object",
"properties": {
"modelVersion": {
"description": "Model Version.",
"minLength": 1,
"type": "string"
},
"captionResult": {
"description": "A brief description of what the image depicts.",
"required": [
"confidence",
"text"
],
"type": "object",
"properties": {
"text": {
"description": "The text of the caption.",
"minLength": 1,
"type": "string"
},
"confidence": {
"format": "double",
"description": "The level of confidence the service has in the caption. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
}
}
},
"denseCaptionsResult": {
"description": "A list of captions.",
"required": [
"values"
],
"type": "object",
"properties": {
"values": {
"description": "A list of captions.",
"type": "array",
"items": {
"description": "A brief description of what the image depicts.",
"required": [
"confidence",
"text"
],
"type": "object",
"properties": {
"text": {
"description": "The text of the caption.",
"minLength": 1,
"type": "string"
},
"confidence": {
"format": "double",
"description": "The level of confidence the service has in the caption. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
},
"boundingBox": {
"description": "A bounding box for an area inside an image.",
"required": [
"h",
"w",
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "Left-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "Top-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"w": {
"format": "int32",
"description": "Width measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
},
"h": {
"format": "int32",
"description": "Height measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
}
}
}
}
}
}
}
},
"metadata": {
"description": "The image metadata information such as height and width.",
"required": [
"height",
"width"
],
"type": "object",
"properties": {
"width": {
"format": "int32",
"description": "The width of the image in pixels.",
"minimum": 1,
"type": "integer"
},
"height": {
"format": "int32",
"description": "The height of the image in pixels.",
"minimum": 1,
"type": "integer"
}
}
},
"tagsResult": {
"description": "A list of tags with confidence level.",
"required": [
"values"
],
"type": "object",
"properties": {
"values": {
"description": "A list of tags with confidence level.",
"type": "array",
"items": {
"description": "An entity observation in the image, along with the confidence score.",
"required": [
"confidence",
"name"
],
"type": "object",
"properties": {
"name": {
"description": "Name of the entity.",
"minLength": 1,
"type": "string"
},
"confidence": {
"format": "double",
"description": "The level of confidence that the entity was observed. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
}
}
}
}
}
},
"objectsResult": {
"description": "Describes detected objects in an image.",
"required": [
"values"
],
"type": "object",
"properties": {
"values": {
"description": "An array of detected objects.",
"type": "array",
"items": {
"description": "Describes a detected object in an image.",
"required": [
"boundingBox",
"tags"
],
"type": "object",
"properties": {
"id": {
"description": "Id of the detected object.",
"minLength": 1,
"type": "string"
},
"boundingBox": {
"description": "A bounding box for an area inside an image.",
"required": [
"h",
"w",
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "Left-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "Top-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"w": {
"format": "int32",
"description": "Width measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
},
"h": {
"format": "int32",
"description": "Height measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
}
}
},
"tags": {
"description": "Classification confidences of the detected object.",
"minItems": 1,
"type": "array",
"items": {
"description": "An entity observation in the image, along with the confidence score.",
"required": [
"confidence",
"name"
],
"type": "object",
"properties": {
"name": {
"description": "Name of the entity.",
"minLength": 1,
"type": "string"
},
"confidence": {
"format": "double",
"description": "The level of confidence that the entity was observed. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
}
}
}
}
}
}
}
}
},
"readResult": {
"description": "The results of an Read operation.",
"required": [
"blocks"
],
"type": "object",
"properties": {
"blocks": {
"description": "A list of text blocks.",
"type": "array",
"items": {
"description": "A detected text block.",
"required": [
"lines"
],
"type": "object",
"properties": {
"lines": {
"description": "List of text lines in the text block.",
"type": "array",
"items": {
"description": "A detected text line.",
"required": [
"boundingPolygon",
"text",
"words"
],
"type": "object",
"properties": {
"text": {
"description": "Text content of the detected text line.",
"minLength": 1,
"type": "string"
},
"boundingPolygon": {
"description": "Bounding polygon of the text line.",
"minItems": 4,
"type": "array",
"items": {
"description": "An object representing a point in the image.",
"required": [
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "The x-coordinate of this point.",
"minLength": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "The y-coordinate of this point.",
"minLength": 0,
"type": "integer"
}
}
}
},
"words": {
"description": "List of words in the text line.",
"type": "array",
"items": {
"description": "A detected word consisting of a contiguous sequence of characters. For non-space delimited languages,\\r\\nsuch as Chinese, Japanese, and Korean, each character is represented as its own word.",
"required": [
"boundingPolygon",
"confidence",
"text"
],
"type": "object",
"properties": {
"text": {
"description": "Text content of the word.",
"minLength": 1,
"type": "string"
},
"boundingPolygon": {
"description": "Bounding polygon of the word.",
"minItems": 4,
"type": "array",
"items": {
"description": "An object representing a point in the image.",
"required": [
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "The x-coordinate of this point.",
"minLength": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "The y-coordinate of this point.",
"minLength": 0,
"type": "integer"
}
}
}
},
"confidence": {
"format": "double",
"description": "The level of confidence that the word was detected. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
}
}
}
}
}
}
}
}
}
}
}
},
"smartCropsResult": {
"description": "Smart cropping result.",
"required": [
"values"
],
"type": "object",
"properties": {
"values": {
"description": "Recommended regions for cropping the image.",
"type": "array",
"items": {
"description": "A region identified for smart cropping. There will be one region returned for each requested aspect ratio.",
"required": [
"aspectRatio",
"boundingBox"
],
"type": "object",
"properties": {
"aspectRatio": {
"format": "double",
"description": "The aspect ratio of the crop region.",
"type": "number"
},
"boundingBox": {
"description": "A bounding box for an area inside an image.",
"required": [
"h",
"w",
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "Left-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "Top-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"w": {
"format": "int32",
"description": "Width measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
},
"h": {
"format": "int32",
"description": "Height measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
}
}
}
}
}
}
}
},
"peopleResult": {
"description": "An object describing whether the image contains people.",
"required": [
"values"
],
"type": "object",
"properties": {
"values": {
"description": "An array of detected people.",
"type": "array",
"items": {
"description": "A person detected in an image.",
"required": [
"boundingBox",
"confidence"
],
"type": "object",
"properties": {
"boundingBox": {
"description": "A bounding box for an area inside an image.",
"required": [
"h",
"w",
"x",
"y"
],
"type": "object",
"properties": {
"x": {
"format": "int32",
"description": "Left-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"y": {
"format": "int32",
"description": "Top-coordinate of the top left point of the area, in pixels.",
"minimum": 0,
"type": "integer"
},
"w": {
"format": "int32",
"description": "Width measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
},
"h": {
"format": "int32",
"description": "Height measured from the top-left point of the area, in pixels.",
"minimum": 1,
"type": "integer"
}
}
},
"confidence": {
"format": "double",
"description": "Confidence score of having observed the person in the image. Confidence scores span the range of 0.0 to 1.0 (inclusive), with higher values indicating a higher confidence of a match.",
"maximum": 1,
"minimum": 0,
"type": "number"
}
}
}
}
}
}
}
}Response 500
Error
Code samples
@ECHO OFF
curl -v -X POST "https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01?features={array}&model-version=latest&language=en&smartcrops-aspect-ratios={array}&gender-neutral-caption=False"
-H "Content-Type: application/json"
-H "Ocp-Apim-Subscription-Key: {subscription key}"
--data-ascii "{body}"
using System;
using System.Net.Http.Headers;
using System.Text;
using System.Net.Http;
using System.Web;
namespace CSHttpClientSample
{
static class Program
{
static void Main()
{
MakeRequest();
Console.WriteLine("Hit ENTER to exit...");
Console.ReadLine();
}
static async void MakeRequest()
{
var client = new HttpClient();
var queryString = HttpUtility.ParseQueryString(string.Empty);
// Request headers
client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", "{subscription key}");
// Request parameters
queryString["features"] = "{array}";
queryString["model-version"] = "latest";
queryString["language"] = "en";
queryString["smartcrops-aspect-ratios"] = "{array}";
queryString["gender-neutral-caption"] = "False";
var uri = "https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01&" + queryString;
HttpResponseMessage response;
// Request body
byte[] byteData = Encoding.UTF8.GetBytes("{body}");
using (var content = new ByteArrayContent(byteData))
{
content.Headers.ContentType = new MediaTypeHeaderValue("< your content type, i.e. application/json >");
response = await client.PostAsync(uri, content);
}
}
}
} // // This sample uses the Apache HTTP client from HTTP Components (http://hc.apache.org/httpcomponents-client-ga/)
import java.net.URI;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
public class JavaSample
{
public static void main(String[] args)
{
HttpClient httpclient = HttpClients.createDefault();
try
{
URIBuilder builder = new URIBuilder("https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01");
builder.setParameter("features", "{array}");
builder.setParameter("model-version", "latest");
builder.setParameter("language", "en");
builder.setParameter("smartcrops-aspect-ratios", "{array}");
builder.setParameter("gender-neutral-caption", "False");
URI uri = builder.build();
HttpPost request = new HttpPost(uri);
request.setHeader("Content-Type", "application/json");
request.setHeader("Ocp-Apim-Subscription-Key", "{subscription key}");
// Request body
StringEntity reqEntity = new StringEntity("{body}");
request.setEntity(reqEntity);
HttpResponse response = httpclient.execute(request);
HttpEntity entity = response.getEntity();
if (entity != null)
{
System.out.println(EntityUtils.toString(entity));
}
}
catch (Exception e)
{
System.out.println(e.getMessage());
}
}
}
<!DOCTYPE html>
<html>
<head>
<title>JSSample</title>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.0/jquery.min.js"></script>
</head>
<body>
<script type="text/javascript">
$(function() {
var params = {
// Request parameters
"features": "{array}",
"model-version": "latest",
"language": "en",
"smartcrops-aspect-ratios": "{array}",
"gender-neutral-caption": "False",
};
$.ajax({
url: "https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01&" + $.param(params),
beforeSend: function(xhrObj){
// Request headers
xhrObj.setRequestHeader("Content-Type","application/json");
xhrObj.setRequestHeader("Ocp-Apim-Subscription-Key","{subscription key}");
},
type: "POST",
// Request body
data: "{body}",
})
.done(function(data) {
alert("success");
})
.fail(function() {
alert("error");
});
});
</script>
</body>
</html>
#import <Foundation/Foundation.h>
int main(int argc, const char * argv[])
{
NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
NSString* path = @"https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01";
NSArray* array = @[
// Request parameters
@"entities=true",
@"features={array}",
@"model-version=latest",
@"language=en",
@"smartcrops-aspect-ratios={array}",
@"gender-neutral-caption=False",
];
NSString* string = [array componentsJoinedByString:@"&"];
path = [path stringByAppendingFormat:@"?%@", string];
NSLog(@"%@", path);
NSMutableURLRequest* _request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:path]];
[_request setHTTPMethod:@"POST"];
// Request headers
[_request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
[_request setValue:@"{subscription key}" forHTTPHeaderField:@"Ocp-Apim-Subscription-Key"];
// Request body
[_request setHTTPBody:[@"{body}" dataUsingEncoding:NSUTF8StringEncoding]];
NSURLResponse *response = nil;
NSError *error = nil;
NSData* _connectionData = [NSURLConnection sendSynchronousRequest:_request returningResponse:&response error:&error];
if (nil != error)
{
NSLog(@"Error: %@", error);
}
else
{
NSError* error = nil;
NSMutableDictionary* json = nil;
NSString* dataString = [[NSString alloc] initWithData:_connectionData encoding:NSUTF8StringEncoding];
NSLog(@"%@", dataString);
if (nil != _connectionData)
{
json = [NSJSONSerialization JSONObjectWithData:_connectionData options:NSJSONReadingMutableContainers error:&error];
}
if (error || !json)
{
NSLog(@"Could not parse loaded json with error:%@", error);
}
NSLog(@"%@", json);
_connectionData = nil;
}
[pool drain];
return 0;
}
<?php
// This sample uses the Apache HTTP client from HTTP Components (http://hc.apache.org/httpcomponents-client-ga/)
require_once 'HTTP/Request2.php';
$request = new Http_Request2('https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01');
$url = $request->getUrl();
$headers = array(
// Request headers
'Content-Type' => 'application/json',
'Ocp-Apim-Subscription-Key' => '{subscription key}',
);
$request->setHeader($headers);
$parameters = array(
// Request parameters
'features' => '{array}',
'model-version' => 'latest',
'language' => 'en',
'smartcrops-aspect-ratios' => '{array}',
'gender-neutral-caption' => 'False',
);
$url->setQueryVariables($parameters);
$request->setMethod(HTTP_Request2::METHOD_POST);
// Request body
$request->setBody("{body}");
try
{
$response = $request->send();
echo $response->getBody();
}
catch (HttpException $ex)
{
echo $ex;
}
?>########### Python 2.7 #############
import httplib, urllib, base64
headers = {
# Request headers
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': '{subscription key}',
}
params = urllib.urlencode({
# Request parameters
'features': '{array}',
'model-version': 'latest',
'language': 'en',
'smartcrops-aspect-ratios': '{array}',
'gender-neutral-caption': 'False',
})
try:
conn = httplib.HTTPSConnection('*.cognitiveservices.azure.com')
conn.request("POST", "/computervision/imageanalysis:analyze?api-version=2024-02-01&%s" % params, "{body}", headers)
response = conn.getresponse()
data = response.read()
print(data)
conn.close()
except Exception as e:
print("[Errno {0}] {1}".format(e.errno, e.strerror))
####################################
########### Python 3.2 #############
import http.client, urllib.request, urllib.parse, urllib.error, base64
headers = {
# Request headers
'Content-Type': 'application/json',
'Ocp-Apim-Subscription-Key': '{subscription key}',
}
params = urllib.parse.urlencode({
# Request parameters
'features': '{array}',
'model-version': 'latest',
'language': 'en',
'smartcrops-aspect-ratios': '{array}',
'gender-neutral-caption': 'False',
})
try:
conn = http.client.HTTPSConnection('*.cognitiveservices.azure.com')
conn.request("POST", "/computervision/imageanalysis:analyze?api-version=2024-02-01&%s" % params, "{body}", headers)
response = conn.getresponse()
data = response.read()
print(data)
conn.close()
except Exception as e:
print("[Errno {0}] {1}".format(e.errno, e.strerror))
####################################require 'net/http'
uri = URI('https://*.cognitiveservices.azure.com/computervision/imageanalysis:analyze?api-version=2024-02-01')
query = URI.encode_www_form({
# Request parameters
'features' => '{array}',
'model-version' => 'latest',
'language' => 'en',
'smartcrops-aspect-ratios' => '{array}',
'gender-neutral-caption' => 'False'
})
if query.length > 0
if uri.query && uri.query.length > 0
uri.query += '&' + query
else
uri.query = query
end
end
request = Net::HTTP::Post.new(uri.request_uri)
# Request headers
request['Content-Type'] = 'application/json'
# Request headers
request['Ocp-Apim-Subscription-Key'] = '{subscription key}'
# Request body
request.body = "{body}"
response = Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
http.request(request)
end
puts response.body