Receipt OCR API Documentation - Mindee Developers

Mindee’s receipt OCR API uses deep learning to automatically, accurately, and instantaneously parse your receipt details. In under a second, the API extracts a set of data from your PDFs or photos of receipts, including:

Expense Locale
Purchase Category
Purchase Subcategory
Document Type
Purchase Date
Purchase Time
Total Amount
Total Net (excluding taxes)
Total Tax
Tip & Gratuity
Taxes
Supplier Name
Supplier Company Registrations
Supplier Address
Supplier Phone Number
Image Orientation
Receipt Number
Line Items

The Receipt OCR API supports handwritten characters for Tip&Gratuity and Total amount fields.

Set up the API

📘
Before making any API calls, you need to have created your API key.

You'll need a receipt. You can use the sample receipt provided below.

Access your Receipt API by clicking on the Expense Receipts card in the APIs Store
From the left navigation, go to documentation > API Reference, you'll find sample code in popular languages and command line.

from mindee import Client, PredictResponse, product

# Init a new client
mindee_client = Client(api_key="my-api-key-here")

# Load a file from disk
input_doc = mindee_client.source_from_path("/path/to/the/file.ext")

# Load a file from disk and parse it.
# The endpoint name must be specified since it cannot be determined from the class.
result: PredictResponse = mindee_client.parse(product.ReceiptV5, input_doc)

# Print a summary of the API result
print(result.document)

# Print the document-level summary
# print(result.document.inference.prediction)

const mindee = require("mindee");
// for TS or modules:
// import * as mindee from "mindee";

// Init a new client
const mindeeClient = new mindee.Client({ apiKey: "my-api-key-here" });

// Load a file from disk
const inputSource = mindeeClient.docFromPath("/path/to/the/file.ext");

// Parse the file
const apiResponse = mindeeClient.parse(
  mindee.product.ReceiptV5,
  inputSource
);

// Handle the response Promise
apiResponse.then((resp) => {
  // print a string summary
  console.log(resp.document.toString());
});

using Mindee;
using Mindee.Input;
using Mindee.Product.Receipt;

string apiKey = "my-api-key-here";
string filePath = "/path/to/the/file.ext";

// Construct a new client
MindeeClient mindeeClient = new MindeeClient(apiKey);

// Load an input source as a path string
// Other input types can be used, as mentioned in the docs
var inputSource = new LocalInputSource(filePath);

// Call the API and parse the input
var response = await mindeeClient
    .ParseAsync<ReceiptV5>(inputSource);

// Print a summary of all the predictions
System.Console.WriteLine(response.Document.ToString());

// Print only the document-level predictions
// System.Console.WriteLine(response.Document.Inference.Prediction.ToString());

#
# Install the Ruby client library by running:
# gem install mindee
#

require 'mindee'

# Init a new client
mindee_client = Mindee::Client.new(api_key: 'my-api-key')

# Load a file from disk
input_source = mindee_client.source_from_path('/path/to/the/file.ext')

# Parse the file
result = mindee_client.parse(
  input_source,
  Mindee::Product::Receipt::ReceiptV5,
  enqueue: true
)

# Print a full summary of the parsed data in RST format
puts result.document

# Print the document-level parsed data
# puts result.document.inference.prediction

import com.mindee.MindeeClient;
import com.mindee.input.LocalInputSource;
import com.mindee.parsing.common.PredictResponse;
import com.mindee.product.receipt.ReceiptV5;
import java.io.File;
import java.io.IOException;

public class SimpleMindeeClient {

  public static void main(String[] args) throws IOException {
    String apiKey = "my-api-key-here";
    String filePath = "/path/to/the/file.ext";

    // Init a new client
    MindeeClient mindeeClient = new MindeeClient(apiKey);

    // Load a file from disk
    LocalInputSource inputSource = new LocalInputSource(filePath);

    // Parse the file
    PredictResponse<ReceiptV5> response = mindeeClient.parse(
        ReceiptV5.class,
        inputSource
    );

    // Print a summary of the response
    System.out.println(response.toString());

    // Print a summary of the predictions
//  System.out.println(response.getDocument().toString());

    // Print the document-level predictions
//    System.out.println(response.getDocument().getInference().getPrediction().toString());

    // Print the page-level predictions
//    response.getDocument().getInference().getPages().forEach(
//        page -> System.out.println(page.toString())
//    );
  }

}

<form onsubmit="mindeeSubmit(event)" >
    <input type="file" id="my-file-input" name="file" />
    <input type="submit" />
</form>

<script type="text/javascript">
    const mindeeSubmit = (evt) => {
        evt.preventDefault()
        let myFileInput = document.getElementById('my-file-input');
        let myFile = myFileInput.files[0]
        if (!myFile) { return }
        let data = new FormData();
        data.append("document", myFile, myFile.name);

        let xhr = new XMLHttpRequest();

        xhr.addEventListener("readystatechange", function () {
            if (this.readyState === 4) {
                console.log(this.responseText);
            }
        });

        xhr.open("POST", "https://api.mindee.net/v1/products/mindee/expense_receipts/v5/predict");
        xhr.setRequestHeader("Authorization", "Token my-api-key-here");
        xhr.send(data);
    }
</script>

curl -X POST \\
  https://api.mindee.net/v1/products/mindee/expense_receipts/v5/predict \\
  -H 'Authorization: Token my-api-key-here' \\
  -H 'content-type: multipart/form-data' \\
  -F document=@/path/to/your/file.png

<?php

use Mindee\Client;
use Mindee\Product\Receipt\ReceiptV5;

// Init a new client
$mindeeClient = new Client("my-api-key-here");

// Load a file from disk
$inputSource = $mindeeClient->sourceFromPath("/path/to/the/file.ext");

// Parse the file
$apiResponse = $mindeeClient->parse(ReceiptV5::class, $inputSource);

echo $apiResponse->document;

Replace my-api-key-here with your new API key, or use the "select an API key" feature and it will be filled automatically.
Copy and paste the sample code of your desired choice in your application, code environment or terminal.
Replace /path/to/my/file with the path to your receipt.

❗️
Always remember to replace your API key!

Run your code. You will receive a JSON response with the receipt details.

API Response

Here is the full JSON response you get when you call the API:

{
  "api_request": {
    "error": {},
    "resources": [
      "document"
    ],
    "status": "success",
    "status_code": 201,
    "url": "https://api.mindee.net/v1/products/mindee/expense_receipts/v5/predict"
  },
  "document": {
    "id": "cc875b84-e83e-4799-aabe-4b2c18f44b27",
    "name": "sample_receipt.jpg",
    "n_pages": 1,
    "is_rotation_applied": true,
    "inference": {
      "started_at": "2021-05-06T16:37:28",
      "finished_at": "2021-05-06T16:37:29",
      "processing_time": 1.121,
      "pages": [
        {
          "id": 0,
          "orientation": {"value": 0},
          "prediction": { .. },
          "extras": {}
        }
      ],
      "prediction": { .. },
      "extras": {}
    }
  }
}

You can find the prediction within the prediction key found in two locations:

In document > inference > prediction for document-level predictions: it contains the different fields extracted at the document level, meaning that for multi-pages PDFs, we reconstruct a single receipt object using all the pages.
In document > inference > pages[ ] > prediction for page-level predictions: it gives the prediction for each page independently. With images, there is only one element on this array, but with PDFs, you can find the extracted data for each PDF page.

Each predicted field may contain one or several values:

a confidence score
a polygon highlighting the information location
a page_id where the information was found (document level only)

{
  "prediction": {
    "category": {
      "confidence": 0.99,
      "value": "food"
    },
    "receipt_number": {
      "confidence": 0.78,
      "polygon": [
        [
          0.12,
          0.32
        ],
        [
          0.22,
          0.32
        ],
        [
          0.22,
          0.48
        ],
        [
          0.12,
          0.48
        ]
      ],
      "value": "R1-002"
    },
    "date": {
      "confidence": 0.99,
      "polygon": [
        [
          0.352,
          0.4
        ],
        [
          0.448,
          0.4
        ],
        [
          0.448,
          0.417
        ],
        [
          0.352,
          0.417
        ]
      ],
      "value": "2022-09-26"
    },
    "document_type": {
      "confidence": 1,
      "value": "EXPENSE RECEIPT"
    },
    "line_items": [
      {
        "confidence": 0.86,
        "description": "article divers",
        "polygon": [
          [
            0.287,
            0.486
          ],
          [
            0.682,
            0.486
          ],
          [
            0.682,
            0.53
          ],
          [
            0.287,
            0.53
          ]
        ],
        "quantity": 165,
        "total_amount": 5.28,
        "unit_price": 0.032
      },
      {
        "confidence": 0.98,
        "description": "sur place",
        "polygon": [
          [
            0.287,
            0.535
          ],
          [
            0.686,
            0.535
          ],
          [
            0.686,
            0.581
          ],
          [
            0.287,
            0.581
          ]
        ],
        "quantity": 1,
        "total_amount": 7.5,
        "unit_price": 7.5
      }
    ],
    "locale": {
      "confidence": 0.97,
      "country": "FR",
      "currency": "EUR",
      "language": "fr",
      "value": "fr-FR"
    },
    "orientation": {
      "confidence": 0.99,
      "degrees": 0
    },
    "subcategory": {
      "confidence": 0.9,
      "value": "shopping"
    },
    "supplier_address": {
      "confidence": 0.95,
      "polygon": [
        [
          0.404,
          0.311
        ],
        [
          0.569,
          0.311
        ],
        [
          0.569,
          0.349
        ],
        [
          0.404,
          0.349
        ]
      ],
      "value": "19 rue de bretagne 75003 paris"
    },
    "supplier_company_registrations": [],
    "supplier_name": {
      "confidence": 0.98,
      "polygon": [
        [
          0.398,
          0.281
        ],
        [
          0.576,
          0.281
        ],
        [
          0.576,
          0.307
        ],
        [
          0.398,
          0.307
        ]
      ],
      "value": "PASTAVINO",
      "raw_value": "Pastavino"
    },
    "supplier_phone_number": {
      "confidence": 0.99,
      "polygon": [
        [
          0.441,
          0.35
        ],
        [
          0.569,
          0.35
        ],
        [
          0.569,
          0.373
        ],
        [
          0.441,
          0.373
        ]
      ],
      "value": "0142772795"
    },
    "taxes": [
      {
        "base": null,
        "code": "TVA",
        "confidence": 0.99,
        "polygon": [
          [
            0.273,
            0.727
          ],
          [
            0.724,
            0.727
          ],
          [
            0.724,
            0.753
          ],
          [
            0.273,
            0.753
          ]
        ],
        "rate": 5.5,
        "value": 0.28
      },
      {
        "base": null,
        "code": "TVA",
        "confidence": 0.96,
        "polygon": [
          [
            0.272,
            0.755
          ],
          [
            0.726,
            0.755
          ],
          [
            0.726,
            0.782
          ],
          [
            0.272,
            0.782
          ]
        ],
        "rate": 10,
        "value": 0.68
      }
    ],
    "time": {
      "confidence": 0.99,
      "polygon": [
        [
          0.454,
          0.397
        ],
        [
          0.527,
          0.397
        ],
        [
          0.527,
          0.416
        ],
        [
          0.454,
          0.416
        ]
      ],
      "value": "13:10"
    },
    "tip": {
      "confidence": 0,
      "polygon": [],
      "value": null
    },
    "total_amount": {
      "confidence": 0.99,
      "polygon": [
        [
          0.654,
          0.644
        ],
        [
          0.717,
          0.644
        ],
        [
          0.717,
          0.668
        ],
        [
          0.654,
          0.668
        ]
      ],
      "value": 12.78
    },
    "total_net": {
      "confidence": 0,
      "polygon": [],
      "value": null
    },
    "total_tax": {
      "confidence": 0.97,
      "polygon": [],
      "value": 0.96
    }
 	}
}

Extracted receipt data

Depending on the field type specified, additional attributes can be extracted from the receipt object. Using the above receipt example the following are the basic fields that can be extracted.

Category
Subcategory
Document type
Receipt Number
Date
Time
Locale & currency
Orientation
Supplier information
Taxes
Total tax
Tip & Gratuity
Total amount
Total net
Line items

Line items

line_items: In the JSON response below, we have the list of items included in the body of the invoice. Each item may contain:

description (String Generic): The description of the item
quantity (number)
unit_price (number)
total_amount (number): The line total amount

{
  "line_items": [
          {
            "confidence": 0.97,
            "description": "Platinum web hosting package Down 35mb, Up 100mb",
            "polygon": [
              [0.035, 0.501],
              [0.965, 0.501],
              [0.965, 0.537],
              [0.035, 0.537]
            ],
            "quantity": 1,
            "total_amount": 65,
            "unit_price": 65
          },
          {
            "confidence": 0.97,
            "description": "2 page website design Includes basic wireframes, and responsive templates",
            "polygon": [
              [0.035, 0.562],
              [0.964, 0.562],
              [0.964, 0.599],
              [0.035, 0.599]
            ],
            "quantity": 3,
            "total_amount": 2100,
            "unit_price": 2100
          },
          {
            "confidence": 0.96,
            "description": "Mobile designs Includes responsive navigation",
            "polygon": [
              [0.035, 0.626],
              [0.964, 0.626],
              [0.964, 0.662],
              [0.035, 0.662]
            ],
            "quantity": 1,
            "total_amount": 250,
            "unit_price": 250
          }
        ]
}

Subcategory

subcategory: The API returns different values depending on the value of the category field.
- If category = transport
  The subcategory field can return values specifying what type of transport the receipt is: plane, train, taxi
- If category = food
  The subcategory field can return values specifying what type of food the receipt is about: restaurant, shopping

{
   "category": {
      "confidence": 0.97,
      "value": "transport",
    },
		"subcategory": {
			"value": "taxi" or "plane" or "restaurant",
			"confidence": 0.54
		}
  }
  
 {
    "category": {
      "confidence": 0.87,
      "value": "food",
    },
		"subcategory": {
			"value": "shopping" or "restaurant",
			"confidence": 0.54
		}
  }

Document type

document_type: is a classification field, that can return two values, either 'expense_receipt' or 'credit_card_receipt'

{
	"document_type": {
		"confidence": 0.77,
		"value": "CREDIT CARD RECEIPT" or "EXPENSE RECEIPT"
	}
}

Receipt number

receipt_number: The identifier assigned to some receipt, analogous to the invoice number.

{
  "receipt_number": {
    "confidence": 0.78,
    "polygon": [[0.12, 0.32], [0.22, 0.32], [0.22, 0.48], [0.12, 0.48]],
    "value": "R1-002"
   }
}

Date

date: The ISO formatted purchase date in the ISO 8601 format.

{
  "date": {
    "confidence": 0.99,
    "page_id": 0,
    "polygon": [[0.479, 0.173], [0.613, 0.173], [0.613, 0.197], [0.479, 0.197]],
    "value": "2016-02-26"
  }
}

Locale & currency

locale: From the receipt, the API can predict where the purchase was made, the language, and the currency. Check the documentation for the latest support. At the time of writing, support is centered on Europe and North America.
- language (String): Language code in ISO 639-1 format as seen on the invoice. The following language codes are supported: ca, de, en, es, fr, it, nl, and pt.
- country (String): Country code in ISO 3166-1 alpha-2 format as seen on the invoice. The following country codes are supported: CA, CH, DE, ES, FR, GB, IT, NL, PT, and US.
- currency (String): Currency code in ISO 4217 format as seen on the receipt. We support the following 44 currencies:

Europe (13)	North America (3)	South America (4)	Middle East (5)	Asia (12)	Africa (6)	Australia (1)	Other
EUR, GBP, CHF, CZK, NOK, SEK, HUF, RON, PLN, RUB, DKK, XPF, TRY	USD, CAD, MXN	COP, BRL, CLP, ARS	AED, SAR, QAR, ILS, OMR	CNY, PHP, SGD, HKD, JPY, MYR, KRW, TWD, THB, VND, INR, IDR	DZD, MAD, TND, XOF, ZAR, XAF	AUD	OTHER

{
  "locale": {
    "confidence": 0.82,
    "country": "GB",
    "currency": "GBP",
    "language": "en",
    "value": "en-GB"
  }
}

Orientation

orientation: The orientation field is only available at the page level as it describes whether the page image should be rotated to be upright. The rotation value is also conveniently available in the JSON response at:
document > inference > pages [ ] > orientation > value.
If the page requires rotation for correct display, the orientation field gives a prediction among these 3 possible outputs:
- 0 degree: the page is already upright
- 90 degrees: the page must be rotated clockwise to be upright
- 270 degrees: the page must be rotated counterclockwise to be upright

{
  "orientation": {
    "confidence": 0.99,
    "degrees": 0
  }
}

📘
All polygon fields across the JSON response are already rotated accordingly!

Supplier Information

supplier-name: In the JSON response below, we have the value and the raw_value of the merchant as found on the receipt.

{
  "supplier_name": {
    "confidence": 0.71,
    "page_id": 0,
    "polygon": [[0.394, 0.068], [0.477, 0.068], [0.477, 0.087], [0.394, 0.087]],
    "value": "CLACHAN",
    "raw_value": "Clachan"
  }
}

supplier_company_registrations: List of detected supplier's company registration number. Each company number object contains an extra attribute:
- type (String Generic): The following company registration numbers are supported: TVA, VAT NUMBER, SIRET, SIREN, NIF, CF, UID, STNR, HRA/HRB, TIN (includes EIN, FEIN, SSN, ATIN, PTIN, ITIN), RFC, BTW, ABN, UEN, CVR, ORGNR, INN, DPH, GSTIN, COMPANY REGISTRATION NUMBER (UK), KVK, DIC, TAX ID, CIF, GST/HST, COC
- value (String): Value of the company identifier

{
  "supplier_company_registrations": [
    {
    "confidence": 0.99,
    "page_id": 0,
    "polygon": [[0.515, 0.962], [0.59, 0.962], [0.59, 0.973], [0.515, 0.973]],
    "type": "SIRET",
    "value": "XXX81125600010"
 		 },

		{
    "confidence": 0.99,
    "page_id": 0,
    "polygon": [[0.658, 0.963], [0.729, 0.963], [0.729, 0.973], [0.658, 0.973]],
    "type": "TVA",
    "value": "FR44837811XXX"
  	}
 ]
}

supplier_address: In the JSON response, we have the value of the supplier address as found on the receipt.

{
  "supplier_address": {
    "confidence": 0.49,
    "page_id": 0,
    "polygon": [[0.756, 0.128], [0.964, 0.128], [0.964, 0.162], [0.756, 0.162]],
    "value": "156 University Ave, Toronto ON, Canada M5H 2H7"
  }
}

Taxes

taxes: In the JSON response below, we have the list of taxes detected in the receipt. Each of these items includes:
- value: the amount of the tax line in the receipt currency
- rate: float between 0 and 1
- base: the amount used to calculate the tax
- code: the tax code. The output is one of the following: VAT, TVA, MWST, IVA, BTW, UST, GST, QST, PST, HST, RST, MERCHANT TAX, GOVERNMENT TAX, TRANSPORTATION TAX, EXCISE TAX, COUNTY TAX, CITY TAX, STATE TAX, LOCAL TAX, SALES TAX, ACCOMMODATION TAX, FOOD TAX, BEVERAGE TAX, FLIGHT TAX, TAX

{
  "taxes": [
    {
      "code": "VAT",
      "confidence": 0.98,
      "page_id": 0,
      "polygon": [[0.19, 0.403], [0.698, 0.403], [0.698, 0.432], [0.19, 0.432]],
      "rate": 20.0,
      "base": 8.50,
      "value": 1.7
    }
  ]
}

Total tax

total_tax: Total tax amount of the purchase.

{
  "total_tax": {
    "confidence": 0.79,
    "page_id": 0,
    "polygon": [[0.549, 0.619], [0.715, 0.619], [0.715, 0.64], [0.549, 0.64]],
    "value": 2.42
  }
}

Tip & Gratuity

tip: Total amount of tip and gratuity. Both typed and handwritten characters are supported.

{
  "tip": {
    "confidence": 0.29,
    "page_id": 0,
    "polygon": [[0.549, 0.619], [0.715, 0.619], [0.715, 0.64], [0.549, 0.64]],
    "value": 12.00
  }
}

Time

time: ISO formatted time as seen on the receipt in HH: MM format.

{
  "time": {
    "confidence": 0.99,
    "page_id": 0,
    "polygon": [[0.62, 0.173], [0.681, 0.173], [0.681, 0.191], [0.62, 0.191]],
    "value": "15:20"
  }
}

Total amount

total_amount: Perhaps the most important part of the receipt is the total spent including taxes, discounts, fees, tips, and gratuity. The Receipt OCR API supports both typed and handwritten characters for this field. The output comes with a confidence score and the bounding box indicating the location on the receipt.

{
  "total_amount": {
    "confidence": 0.99,
    "page_id": 0,
    "polygon": [[0.549, 0.619], [0.715, 0.619], [0.715, 0.64], [0.549, 0.64]],
    "value": 10.2
  }
}

Total net (excluding taxes)

total_net: Total amount of the purchase excluding taxes.

{
  "total_net": {
    "confidence": 0.89,
    "page_id": 0,
    "polygon": [[0.549, 0.619], [0.715, 0.619], [0.715, 0.64], [0.549, 0.64]],
    "value": 14.27
  }
}

Questions?
Join our Slack

Set up the API

📘

❗️

API Response

Extracted receipt data

Line items

Category

Subcategory

Document type

Receipt number

Date

Locale & currency

Orientation

📘

Supplier Information

Taxes

Total tax

Tip & Gratuity

Time

Total amount

Total net (excluding taxes)