DEV Community: xulihang

How to Build a Document Scanner with Expo

xulihang — Fri, 10 Nov 2023 02:53:09 +0000

Expo is an open-source platform for making universal native apps for Android, iOS, and the web with JavaScript and React. It is basically a set of tools built on top of React Native, which makes it easy to develop and distribute apps.

In this article, we are going to build a document scanner with Expo. It can detect document boundaries and crop the document image via the camera and scan documents via physical document scanners.

Demo video

SDKs Used

Dynamsoft Document Normalizer which provides the ability to detect document boundaries and perform perspective correction.
Dynamsoft Service from Dynamic Web TWAIN which provides REST APIs for accessing document scanners.

New Project

Create a new Expo project:
```
npx create-expo-app DocumentScanner
```

Add camera permissions in app.json:

"ios": {
  "infoPlist": {
    "NSCameraUsageDescription": "This app uses the camera to scan barcodes."
  }
},
"android": {
  "permissions": ["android.permission.CAMERA","android.permission.INTERNET"]
}

Add dependencies:

npx expo install react-native-webview expo-camera react-native-safe-area-context expo-sharing expo-file-system

Design the Home Page

Update App.js to add an image element to display the scanned image, a button to scan documents, a button to share the document image, a button to view the history and two selections for choosing the scanning device and the color mode.

There are three color modes: black & white, gray and color. The device array includes the camera and connected document scanners.

import { StatusBar } from 'expo-status-bar';
import { useState,useEffect, useRef } from 'react';
import { StyleSheet, View, Image, Text } from 'react-native';
import Button from './components/Button';
import Select from './components/Select';
import { SafeAreaView, SafeAreaProvider  } from 'react-native-safe-area-context';

const PlaceholderImage = require('./assets/thumbnail.png');
const colorModes = ["Black&White","Gray","Color"];

export default function App() {
  const path = useRef("");
  const [devices,setDevices] = useState(["Camera"]);
  const [selectedDeviceIndex,setSelectedDeviceIndex] = useState(0);
  const [selectedColorMode,setSelectedColorMode] = useState("Color");
  const [image,setImage] = useState(PlaceholderImage);

  const renderBody = () => {
    return (
      <View style={styles.home}>
        <View style={styles.imageContainer}>
          <Image source={image} style={styles.image} />
        </View>
        <View style={styles.footerContainer}>
          <View style={styles.option}>
            <Text style={styles.label}>Device:</Text>
            <Select style={styles.select} label={devices[selectedDeviceIndex]}></Select>
          </View>
          <View style={styles.option}>
            <Text style={styles.label}>Color Mode:</Text>
            <Select style={styles.select} label={selectedColorMode} ></Select>
          </View>
          <View style={styles.buttons}>
            <View style={styles.buttonContainer}>
              <Button label="Scan" onPress={()=>scan()} />
            </View>
            <View style={styles.buttonContainer}>
              <Button style={styles.button} label="Share" onPress={()=>share()} />
            </View>
          </View>
          <View>
            <Button style={styles.button} label="History"/>
          </View>
        </View>
      </View>
    )
  }

  return (
    <SafeAreaProvider>
      <SafeAreaView style={styles.container}>
        {renderBody()}
        <StatusBar style="auto"/>
      </SafeAreaView>
    </SafeAreaProvider>
  );
}

const styles = StyleSheet.create({
  container: {
    flex: 1,
  },
  button:{
    marginBottom: 5,
  },
  buttons: {
    flexDirection:"row",
  },
  buttonContainer:{
    width:"50%",
  },
  home: {
    flex: 1,
    width: "100%",
    backgroundColor: '#25292e',
    alignItems: 'center',
  },
  footerContainer: {
    flex: 3 / 5,
    width: "100%",
  },
  option:{
    flexDirection:"row",
    alignItems:"center",
    marginHorizontal: 20,
    height: 40,
  },
  label:{
    flex: 3 / 7,
    color: 'white', 
    marginRight: 10,
  },
  select:{
    flex: 1,
  },
  imageContainer: {
    flex: 1,
    paddingTop: 20,
    alignItems:"center",
  },
  image: {
    width: 320,
    height: "95%",
    borderRadius: 18,
    resizeMode: "contain",
  },
});

Two custom components are defined as well.

components/Button.js:

import { StyleSheet, View, Pressable, Text } from 'react-native';

export default function Button({ label, onPress }) {
  return (
    <View
      style={[styles.buttonContainer, { borderWidth: 3, borderColor: "#ffd33d", borderRadius: 18 }]}
      >
        <Pressable
          style={[styles.button, { backgroundColor: "#fff" }]}
          onPress={onPress}
        >
          <Text style={[styles.buttonLabel, { color: "#25292e" }]}>{label}</Text>
        </Pressable>
    </View>
  );
}

const styles = StyleSheet.create({
  buttonContainer: {
    width: "auto",
    marginHorizontal: 10,
    height: 40,
    alignItems: 'center',
    justifyContent: 'center',
    padding: 3,
    margin: 3,
  },
  button: {
    borderRadius: 10,
    width: '100%',
    height: '100%',
    alignItems: 'center',
    justifyContent: 'center',
    flexDirection: 'row',
  },
  buttonIcon: {
    paddingRight: 8,
  },
  buttonLabel: {
    color: '#fff',
    fontSize: 16,
  },
});

components/Select.js:

import { StyleSheet, View, Pressable, Text } from 'react-native';

export default function Select({ label, onPress }) {
  return (
    <View
      style={[styles.container]}
      >
        <Pressable
          style={[styles.button]}
          onPress={onPress}
        >
          <Text ellipsizeMode="tail" numberOfLines={1} style={[styles.label]}>{label}</Text>
        </Pressable>
    </View>
  );
}

const styles = StyleSheet.create({
  container: {
    flex:1,
    borderWidth: 1, 
    borderColor: "white", 
    borderRadius: 10,
    height: 30,
    justifyContent:"center",
  },
  label:{
    marginLeft: 10,
    color: "white",
  }
});

Define an Items Picker Component

Create a new component for picking an item. We can use it to pick which device to use, which color mode to use and which action to take, etc.

{% raw %}import { View, Text, Pressable, StyleSheet } from 'react-native';

export default function ItemsPicker({ items,onPress }) {
  return (
    <View style={styles.container}>
      <Text style={{color: "white"}}>Select an item:</Text>
      {items.map((item, idx) => (
        <Pressable key={idx} onPress={()=>onPress(item,idx)}>
          <Text style={styles.item}>{item}</Text>
        </Pressable>
      ))}
    </View>
  );
}

const styles = StyleSheet.create({
  container: {
    flex:1,
    paddingTop: 20,
    paddingLeft: 20,
    alignItems: 'flex-start',
    backgroundColor: 'black',
  },
  item: {
    color: "white",
    padding: 10,
    fontSize: 18,
    height: 44,
  },
});{% endraw %}

Then, in App.js we can use the ItemsPicker to configure the scanning.

const [showDevicePicker,setShowDevicePicker] = useState(false);
const [showColorModePicker,setShowColorModePicker] = useState(false);
const renderBody = () => {
  if (showDevicePicker) {
    return (
      <ItemsPicker items={devices} onPress={(device,idx) => {
        console.log(device);
        setSelectedDeviceIndex(idx);
        setShowDevicePicker(false);
      }}></ItemsPicker>
    )
  }
  if (showColorModePicker) {
    return (
      <ItemsPicker items={colorModes} onPress={(mode) => {
        setSelectedColorMode(mode);
        setShowColorModePicker(false);
      }}></ItemsPicker>
    )
  }
}

Scan Documents from the Camera

Next, we are going the integrate scanning documents from the camera.

Create a new component named DocumentScanner.js under components.

import { StyleSheet, View } from 'react-native';
import { useEffect,useState } from 'react';
export default function DocumentScanner(props) {
    return <View/>
  }
}

const styles = StyleSheet.create({
});

Request camera permission with expo-camera when the component is mounted.

const [hasPermission, setHasPermission] = useState(null);
useEffect(() => {
  (async () => {
    const { status } = await Camera.requestCameraPermissionsAsync();
    setHasPermission(status === "granted");
  })();
}, []);

Use react-native-webview to load a document scanning web app built with Dynamsoft Camera Enhancer and Dynamsoft Document Normalizer if the camera permission is granted. The web app is built for use in react-native-webview. It can post the scanned document image as data URL to the react native app. We can pass URL params to configure its behaviors like the color mode and the license. A license is needed to use Dynamsoft Document Normalizer. You can apply for a license here.

{----% raw %----}const getURI = () => {
  let URI = 'https://tony-xlh.github.io/Vanilla-JS-Document-Scanner-Demos/react-native/?autoStart=true';
  if (props.colorMode == "Black&White") {
    URI = URI + "&colorMode="+0;
  }else if (props.colorMode == "Gray"){
    URI = URI + "&colorMode="+1;
  }else{
    URI = URI + "&colorMode="+2;
  }
  if (props.license) {
    URI = URI + "&license="+props.license;
  }
  return URI;
}
if (hasPermission) {
  return (
    <WebView
      style={styles.webview}
      allowsInlineMediaPlayback={true}
      mediaPlaybackRequiresUserAction={false}
      onMessage={(event) => {
        if (!event.nativeEvent.data) {
          if (props.onClosed) {
            props.onClosed();
          }
        }else{
          if (props.onScanned) {
            const dataURL = event.nativeEvent.data;
            props.onScanned(dataURL);
          }
        }
      }}
      source={{ uri: getURI() }}
    />
  );
}else{
  return <Text>No permission.</Text>
}{----% endraw %----}

Show the document scanner in the home page after the scan button is pressed and the selected device is camera. It will capture an image automatically if it detects three overlapped document regions consecutively. The image will be saved to the app's document directory and be displayed in the page.

const path = useRef("");
const [showScanner,setShowScanner] = useState(false);
const onScanned = async (dataURL) => {
  const timestamp = new Date().getTime();
  path.current = FileSystem.documentDirectory + timestamp + ".png";
  const base64Code = removeDataURLHead(dataURL);
  await FileSystem.writeAsStringAsync(path.current, base64Code, {
    encoding: FileSystem.EncodingType.Base64,
  });
  setImage({uri: path.current});
  setShowScanner(false);
}

const removeDataURLHead = (dataURL) => {
  return dataURL.substring(dataURL.indexOf(",")+1,dataURL.length);
}

const renderBody = () => {
  if (showScanner) {
    return (
      <DocumentScanner 
        license="DLS2eyJoYW5kc2hha2VDb2RlIjoiMTAwMjI3NzYzLVRYbFhaV0pRY205cSIsIm1haW5TZXJ2ZXJVUkwiOiJodHRwczovL21sdHMuZHluYW1zb2Z0LmNvbSIsIm9yZ2FuaXphdGlvbklEIjoiMTAwMjI3NzYzIiwic3RhbmRieVNlcnZlclVSTCI6Imh0dHBzOi8vc2x0cy5keW5hbXNvZnQuY29tIiwiY2hlY2tDb2RlIjotMzg1NjA5MTcyfQ=="
        colorMode={selectedColorMode} 
        onScanned={(dataURL)=>onScanned(dataURL)}
      ></DocumentScanner>
    )
  }
}

Screenshot:

Here, we use react-native-webview to integrate the camera document scanning ability for ease of use as it can run in Expo Go. You can use the native module if you want to integrate the ability natively.

Scan Documents from Document Scanners

Next, we are going to scan documents from document scanners through Dynamsoft Service's REST API.

Install Dynamsoft Service on a PC which is connected to document scanners and make it accessible in an intranet. You can configure its IP on the configuration page and you can find the download links here.

Create a DynamsoftService class for getting the scanners list and acquiring images from scanners via the REST API.

export class DynamsoftService {
  endpoint;
  license;
  constructor(endpoint,license) {
    this.endpoint = endpoint;
    this.license = license;
  }

  async getDevices(){
    const url = this.endpoint + "/DWTAPI/Scanners";
    const response = await fetch(url, {"method":"GET", "mode":"cors", "credentials":"include"});
    let scanners = await response.json();
    return scanners;
  }

  async acquireImage(device,pixelType){
    let url = this.endpoint + "/DWTAPI/ScanJobs";
    let scanParams = {
      license:this.license
    };
    if (device) {
      // optional. use the latest device.
      scanParams.device = device;
    }
    scanParams.config = {
      IfShowUI: false,
      Resolution: 200,
      IfFeederEnabled: false,
      IfDuplexEnabled: false,
    };
    scanParams.caps = {};
    scanParams.caps.exception = "ignore";
    scanParams.caps.capabilities = [
      {
        capability: 257, // pixel type
        curValue: pixelType
      }
    ]
    const response = await fetch(url, {"body": JSON.stringify(scanParams), "method":"POST", "mode":"cors", "credentials":"include"});
    if (response.status == 201)
    {
      curJobid = await response.text();
      return (await this.getImage(curJobid));
    }else{
      let message = await response.text();
      throw new Error(message);
    }
  }

  async getImage(jobid) {
    // get image.
    const url = this.endpoint + "/DWTAPI/ScanJobs/" + jobid + '/NextDocument';
    const response = await fetch(url, {"method":"GET", "mode":"cors", "credentials":"include"});
    if (response.status === 200)
    {
      const image = await response.blob();
      return this.blobToBase64(image);
    }
  }

  blobToBase64( blob ) {
    return new Promise((resolve, reject) => {
      const reader = new FileReader();
      reader.readAsDataURL(blob);
      reader.onloadend = () => {
        const uri = reader.result?.toString();
        resolve(uri);
      };  
    })
  }
}

In App.js, fetch the list of scanners via the REST API and add them to the device select. A license is needed to use it. You can apply for a license here.

export default function App() {
  const service = useRef();
  const scanners = useRef();
  const [devices,setDevices] = useState(["Camera"]);
  useEffect(() => {
    service.current = new DynamsoftService("http://192.168.8.65:18622","LICENSE");
    fetchDevicesList();
  }, []);
  const fetchDevicesList = async () =>{
    scanners.current = await service.current.getDevices();
    let newDevices = ["Camera"];
    for (let index = 0; index < scanners.current.length; index++) {
      const scanner = scanners.current[index];
      newDevices.push(scanner.name);
    }
    setDevices(newDevices);
  }
}

Acquire a document image after the scan button is pressed.

const scan = async () => {
  if (selectedDeviceIndex == 0) {
    setShowScanner(true);
  }else{
    setModalVisible(true);
    const selectedScanner = scanners.current[selectedDeviceIndex - 1];
    const pixelType = colorModes.indexOf(selectedColorMode);
    const image = await service.current.acquireImage(selectedScanner.device,pixelType);
    onScanned(image);
    setModalVisible(false);
  }
}

A modal will be shown during the scanning process.

const [modalVisible, setModalVisible] = useState(false);
return (
  <View style={styles.home}>
    <Modal
      transparent={true}
      visible={modalVisible}
      >
      <View style={styles.centeredView}>
        <View style={styles.modalView}>
          <Text>Scanning...</Text>
        </View>
      </View>
    </Modal>
  </View>
)

Screenshot:

Manage History

Add a history browser component to manage scanned documents under components/HistoryBrowser.js.

{% raw %}import { Alert, StyleSheet, View, Text, FlatList, Image,Button,Pressable,Dimensions } from 'react-native';
import { useEffect,useState,useRef } from 'react';
import ItemsPicker from './ItemsPicker';
import * as Sharing from 'expo-sharing';
import * as FileSystem from 'expo-file-system';

const width = Dimensions.get('window').width;
const height = Dimensions.get('window').height;
const actions = ["Delete","Share","Get info","Cancel"];

export default function HistoryBrowser(props) {
  const [images,setImages] = useState([]);
  const [showActionPicker,setShowActionPicker] = useState(false);
  const pressedImageName = useRef("");
  useEffect(() => {
    console.log(width);
    console.log(height);
    readImagesList();
  }, []);

  const readImagesList = async () => {
    let newImages = [];
    const files = await FileSystem.readDirectoryAsync(FileSystem.documentDirectory);
    for (let index = 0; index < files.length; index++) {
      const file = files[index];
      if (file.toLowerCase().endsWith(".png")) {
        newImages.push(file);
      }
    }
    setImages(newImages);
  }

  const getURI = (filename) => {
    const uri = FileSystem.documentDirectory + filename;
    return uri;
  }

  const goBack = () => {
    if (props.onBack) {
      props.onBack();
    }
  }

  const deleteFile = async () => {
    if (pressedImageName.current != "") {
      setImages([]);
      const path = FileSystem.documentDirectory + pressedImageName.current;
      await FileSystem.deleteAsync(path);
      pressedImageName.current = "";
      readImagesList();
    }
  }

  const share = () => {
    if (pressedImageName.current != "") {
      const path = FileSystem.documentDirectory + pressedImageName.current;
      Sharing.shareAsync(path);
    }
  }

  const getInfo = async () => {
    if (pressedImageName.current != "") {
      const path = FileSystem.documentDirectory + pressedImageName.current;
      const info = await FileSystem.getInfoAsync(path);
      const time = new Date(info.modificationTime*1000);
      let message = "Time: " + time.toUTCString() + "\n";
      message = message + "Size: " + info.size/1000 + "KB";
      Alert.alert(pressedImageName.current,message);

    }

  }

  return (
    <View style={styles.container}>
      {showActionPicker && (
        <View style={styles.pickerContainer}>
          <ItemsPicker items={actions} onPress={(action) => {
            console.log(action);
            setShowActionPicker(false);
            if (action === "Delete") {
              deleteFile();
            }else if (action === "Share"){
              share();
            }else if (action === "Get info"){
              getInfo();
            }
          }}></ItemsPicker>
        </View>
      )
      }
      <View style={styles.backButtonContainer} >
        <Button title="< Back" onPress={goBack}></Button>  
      </View>
      <FlatList
        horizontal={true}
        style={styles.flat}
        data={images}
        renderItem={({item}) => 
          <Pressable onPress={()=>{
            pressedImageName.current = item;
            setShowActionPicker(true);
          }}>
            <Image style={styles.image} source={{
              uri: getURI(item),
            }}/>
          </Pressable>}
      />
    </View>
  )
}

const styles = StyleSheet.create({
  container:{
    flex: 1,
    backgroundColor: "#25292e",
    alignItems:"center",
  },
  image: {
    width: width*0.9,
    height: height*0.9,
    resizeMode: "contain",
  },
  pickerContainer:{
    position: "absolute",
    top: 0,
    left: 0,
    width: "100%",
    height: "100%",
    zIndex: 20,
  },
  backButtonContainer:{
    position:"absolute",
    top: 0,
    left: 0,
    zIndex: 10,
  }
});{% endraw %}

Source Code

We've now completed the demo. Get the source code and have a try: https://github.com/tony-xlh/expo-document-scanner

How to Build a Document Scanning Component with LitElement

xulihang — Mon, 06 Nov 2023 07:33:20 +0000

Lit is a simple library for building fast, lightweight web components. At Lit's core is a boilerplate-killing component base class that provides reactive state, scoped styles, and a declarative template system that's tiny, fast and expressive.

In this article, we are going to build a web component with LitElement to scan documents in an HTML website based on Dynamic Web TWAIN.

New Project

Although Lit can be used without a build system, in this article, we are going to use webpack with the following template (clone with git):

git clone https://github.com/wbkd/webpack-starter

Install Dependencies

Install Lit.

   npm install lit

Install Dynamic Web TWAIN.

   npm install dwt

In addition, we need to copy the resources of Dynamic Web TWAIN to the public folder.

Install ncp.
```
  npm install --save-dev ncp
```

Modify package.json to copy the resources for the build and start commands.

   "scripts": {
     "lint": "npm run lint:styles; npm run lint:scripts",
     "lint:styles": "stylelint src",
     "lint:scripts": "eslint src",
     "build": "cross-env NODE_ENV=production webpack --config webpack/webpack.config.prod.js",
     "start": "webpack serve --config webpack/webpack.config.dev.js"
  +  "build": "ncp node_modules/dwt/dist public/dwt-resources && cross-env NODE_ENV=production webpack --config webpack/webpack.config.prod.js",
  +  "start": "ncp node_modules/dwt/dist public/dwt-resources && webpack serve --config webpack/webpack.config.dev.js"
   },

Modify webpack.common.js to copy the files in the public folder to the output folder instead of the public folder inside the output folder.

   new CopyWebpackPlugin({
  -  patterns: [{ from: Path.resolve(__dirname, '../public'), to: 'public' }],
  +  patterns: [{ from: Path.resolve(__dirname, '../public'), to: '' }],
   }),

Write a Document Scanner Component

Create a new documentscanner.js file under src\scripts with the following template.

   import {LitElement, html, css} from 'lit';

   export class DocumentScanner extends LitElement {
     static properties = {
     };
     DWObject;
     static styles = css`
       :host {
         display: block;
       }
       `;
     constructor() {
       super();
     }

     render() {
       return html``;
     }
   }
   customElements.define('document-scanner', DocumentScanner);

Add a div element as the container for the controls of Dynamic Web TWAIN (mainly to view documents).

   render() {
     return html`<div id="dwtcontrolContainer"></div>`;
   }

Configure Dynamic Web TWAIN in the constructor. You need a license to use Dynamic Web TWAIN. You can apply for a license here.

   constructor() {
     super();
     Dynamsoft.DWT.AutoLoad = false;
     Dynamsoft.DWT.ResourcesPath = "/dwt/dist";
     Dynamsoft.DWT.ProductKey = "DLS2eyJoYW5kc2hha2VDb2RlIjoiMjAwMDAxLTE2NDk4Mjk3OTI2MzUiLCJvcmdhbml6YXRpb25JRCI6IjIwMDAwMSIsInNlc3Npb25QYXNzd29yZCI6IndTcGR6Vm05WDJrcEQ5YUoifQ=="; //one-day trial
   }

Initialize Dynamic Web TWAIN in the firstUpdated lifecycle after the dom is first updated and bind it to the container in the previous step. In addition, dispatch a custom event with the object of Dynamic Web TWAIN so that we can control it in a parent node.

   DWObject;
   firstUpdated() {
     let pThis = this;
     let dwtContainer = this.renderRoot.getElementById("dwtcontrolContainer");
     Dynamsoft.DWT.CreateDWTObjectEx(
       {
         WebTwainId: 'dwtcontrol'
       },
       function(obj) {
         pThis.DWObject = obj;
         pThis.DWObject.Viewer.bind(dwtContainer);
         pThis.DWObject.Viewer.show();
         pThis.DWObject.Viewer.width = "100%";
         pThis.DWObject.Viewer.height = "100%";
         const event = new CustomEvent('initialized', {
           detail: {
             DWObject: pThis.DWObject
           }
         });
         pThis.dispatchEvent(event);
       },
       function(err) {
         console.log(err);
       }
     );
   }

Add one button to scan documents and one button to save the document images as a PDF file.

   render() {
     return html`
     <div class="buttons">
       <button @click=${this.scan}>Scan</button>
       <button @click=${this.save}>Save</button>
     </div>
     <div id="dwtcontrolContainer"></div>`;
   }
   scan(){
     let pThis = this;
     if (pThis.DWObject) {
       pThis.DWObject.SelectSource(function () {
         pThis.DWObject.OpenSource();
         pThis.DWObject.AcquireImage();
       },
         function () {
           console.log("SelectSource failed!");
         }
       );
     }
   }

   save(){
     if (this.DWObject) {
       this.DWObject.SaveAllAsPDF("Scanned.pdf");
     }
   }

Add a reactive property named total to reflect how many documents are scanned. We can update its value in the OnBufferChanged event of Web TWAIN.

   export class DocumentScanner extends LitElement {
     static properties = {
       total: {},
     };
     constructor() {
       super();
       this.total = 0;
       //...
     }
     render() {
       return html`
       <div class="buttons">
         <button @click=${this.scan}>Scan</button>
         <button @click=${this.save}>Save</button>
       </div>
       <div id="dwtcontrolContainer"></div>
       <div class="status">Total: ${this.total}</div>`;
     }

     firstUpdated() {
       //...
       Dynamsoft.DWT.CreateDWTObjectEx(
         {
           WebTwainId: 'dwtcontrol'
         },
         function(obj) {
           //...
           pThis.DWObject.RegisterEvent('OnBufferChanged',function () {
             pThis.total = pThis.DWObject.HowManyImagesInBuffer;
           });
           //...
         },
         function(err) {
           console.log(err);
         }
       );
     } 
   }

Set the styles for the component.

   static styles = css`
     :host {
       display: block;
     }
     .buttons {
       height: 25px;
     }
     #dwtcontrolContainer {
       width: 100%;
       height: calc(100% - 50px);
     }
     .status {
       height: 25px;
     }
     `;

Use the Document Scanner Component

Import the component in the index.js file.

   import { DocumentScanner } from './documentscanner'; // eslint-disable-line

Add the component in the index.html.

   <document-scanner 
     style="width:320px;height:480px;"
   ></document-scanner>

All right, we can now scan documents from the browser.

Online demo

Source Code

Get the source code of the demo to have a try:

https://github.com/tony-xlh/document-scanner-lit-element

How to Build a Quasar QR Code Scanner with Capacitor

xulihang — Thu, 01 Sep 2022 07:26:37 +0000

Quasar is a Vue.js-based framework, which allows web developers to quickly create responsive websites/apps, while Capacitor is a runtime for creating cross-platform iOS, Android, and Progressive Web Apps with JavaScript, HTML, and CSS.

In this article, we are going to use the two to create a QR code scanner using the capacitor plugin of Dynamsoft Barcode Reader. The app can run as Web, Android and iOS apps.

Online demo.

Check out the Ionic Vue article, if you prefer to use the Ionic framework.

Build a Quasar QR Code Scanner using Capacitor

New Project

Install quasar cli:

npm install -g @quasar/cli

Then, use it to create a new project:

npm init quasar

Because it has to run in Android's webview, for compatibility concerns, we choose Quasar v1 + Vue.js 2 + Composition API to build this app.

√ What would you like to build? » App with Quasar CLI, let's go!
√ Project folder: ... quasar-qrcode-scanner
√ Pick Quasar version: » Quasar v1 (Vue 2)
√ Pick script type: » Typescript
√ Package name: ... quasar-qrcode-scanner
√ Project product name: (must start with letter if building mobile apps) ... Quasar QR Code Scanner
√ Project description: ... A Quasar Project
√ Author: ...
√ Pick a Vue component style: » Composition API (recommended) (https://github.com/vuejs/composition-api)
√ Pick your CSS preprocessor: » Sass with SCSS syntax
√ Pick a Quasar components & directives import strategy: (can be changed later) » * Auto-import in-use Quasar components & directives
        - also treeshakes Quasar; minimum bundle size
√ Check the features needed for your project: » ESLint
√ Pick an ESLint preset: » Prettier

Install Dependencies

npm install capacitor-plugin-dynamsoft-barcode-reader

Modify the Default Files

Remove the example components under src/components.
Open src/layouts/MainLayout.vue, simplify its template with the following content:

   <template>
     <q-layout view="lHh Lpr lFf">
       <q-header elevated >
         <q-toolbar>
           <q-toolbar-title>
             QR Code Scanner
           </q-toolbar-title>
         </q-toolbar>
       </q-header>
       <q-page-container>
         <router-view />
       </q-page-container>
     </q-layout>
   </template>

Open src/pages/Index.vue, add a floating action button to start the scanner page to scan QR codes and a QList to display the results. Users can clikc the item to copy the result.

   <template>
     <q-page class="row justify-evenly">
       <div class="full">
         <q-list v-if="results.barcodeResults.value.length>0" dense bordered separator padding class="rounded-borders">
           <q-item @click="copy(result.barcodeText)" clickable v-ripple v-for="(result, index) in results.barcodeResults.value" :key="index">
             <q-item-section>
               <q-item-label :lines="1">{{ result.barcodeText }}</q-item-label>
               <q-item-label caption>{{ result.barcodeFormat }}</q-item-label>
             </q-item-section>
           </q-item>
         </q-list>
         <q-page-sticky position="bottom-left" :offset="[18,18]">
           <q-btn @click="goScan" fab icon="camera_alt" color="blue" />
         </q-page-sticky>
       </div>
     </q-page>
   </template>

The goScan and copy functions:

   import { Clipboard } from '@capacitor/clipboard';
   import { Notify } from 'quasar';
   const goScan = async () => {
     await router.push('/scanner');
   }

   const copy = async (text:string) => {
     await Clipboard.write({
       string: text
     });
     Notify.create({
       message: 'Copied.'
     });
   }

Use vue-router with the composition API to navigate to another page.

The dependent vue-router of the current version quasar does not have useRoute and useRouter to use in Vue 2.7 with the composition API. We can use the following workaround (GitHub issue).

Create a file under utils/index.js.

  import { getCurrentInstance } from 'vue'

  export function useRoute() {
    const { proxy } = getCurrentInstance()
    const route = proxy.$route
    return route
  }
  export function useRouter() {
    const { proxy } = getCurrentInstance()
    const router = proxy.$router
    return router
  }

In the index page, use it to navigate to the scanner page.

  import { useRouter } from '../utils/index.js'
  export default defineComponent({
    name: 'PageIndex',
    components: {},
    setup() {
      const router = useRouter();
      const goScan = async () => {
        await router.push('/scanner');
      }
      return {goScan};
    }
  });

Screenshot of the index page:

Add a Scanner Page

Create a new file named Scanner.vue under src\pages.
Open src/router/routes.ts to register the route for the scanner page.

   {
     path: '/scanner',
     component: () => import('pages/Scanner.vue')
   },

Add a QR Code Scanner component. We've defined the component in the previous Ionic Vue article. Although that project is in Vue 3, we can still directly use the component in a Vue 2 project with the composition api enabled. We are going to put the vue file under src/components/QRCodeScanner.vue.
Use the QR Code Scanner component in the scanner page.

   <template>
     <q-layout view="lHh Lpr lFf">
       <QRCodeScanner
         license="DLS2eyJoYW5kc2hha2VDb2RlIjoiMjAwMDAxLTE2NDk4Mjk3OTI2MzUiLCJvcmdhbml6YXRpb25JRCI6IjIwMDAwMSIsInNlc3Npb25QYXNzd29yZCI6IndTcGR6Vm05WDJrcEQ5YUoifQ=="
         :torchOn="torchOn"
         :zoomFactor="zoomFactor"
         :runtimeSettings="runtimeSettings"
         @onScanned="onScanned"
         @onPlayed="onPlayed"
       ></QRCodeScanner>
     </q-layout>
   </template>
   <script lang="ts">

   export default defineComponent({
     name: 'PageScanner',
     components: { QRCodeScanner },
     setup() {
       const scanned = ref(false);
       const zoomFactor = ref(1.0);
       const torchOn = ref(false);
       const runtimeSettings = ref('');
       const router = useRouter();

       const onPlayed = (resolution:string) => {
         console.log(resolution);
       }
       const onScanned = (result:ScanResult) => {
         console.log(result);
       };

       onMounted(() => {
         zoomFactor.value = 1.0;
         App.addListener("backButton",() => {
           router.go(-1);
         });
       });

       return {onPlayed, onScanned, goBack, zoomFactor, runtimeSettings};
     }});
    </script>

Add a float action button for cancelling the scanner and controlling the torch and zoom status.

Template:

   <q-page-sticky style="z-index:1;" position="bottom-left" :offset="[18,18]">
     <q-fab color="blue" icon="keyboard_arrow_up" direction="up">
       <q-fab-action @click="toggleTorch" color="blue" icon="flashlight_on" />
       <q-fab-action @click="zoomIn" color="blue" icon="add" />
       <q-fab-action @click="zoomOut" color="blue" icon="remove" />
       <q-fab-action @click="goBack" color="blue" icon="arrow_back" />
     </q-fab>
   </q-page-sticky>

Script:

   const zoomFactor = ref(1.0);
   const torchOn = ref(false);
   const toggleTorch = () => {
      torchOn.value = ! torchOn.value;
    };

   const zoomIn = () => {
     zoomFactor.value = zoomFactor.value + 0.3;
   };

   const zoomOut = () => {
     zoomFactor.value = Math.max(zoomFactor.value - 0.3, 1.0);
   };

   const goBack = () => {
     update([]);
     router.go(-1);
   }

Screenshot of the scanner page:

Create a Store for Passing the QR Code Results to the Index Page

We can create a store to share data between different pages using Vuex.

Create a new Vuex store named barcodes.

   quasar new store barcodes

It will add the following files to the project:

   .
   └── src/
       └── store/
           ├── index.js         # Vuex Store definition
           └── barcodes         # Module "barcodes"
               ├── index.js     # Gluing the module together
               ├── actions.js   # Module actions
               ├── getters.js   # Module getters
               ├── mutations.js # Module mutations
               └── state.js     # Module state

Edit src/store/index.js to add a reference to the new module.

   export default function (/* { ssrContext } */) {
     const Store = new Vuex.Store({
       modules: {
   +     barcodes
       },

       strict: process.env.DEBUGGING
     })

     return Store
   }

Update state.js to define the states and mutations.js to define the setter.

   // src/store/barcodes/mutations.js
   export function update(state, results) {
     state.barcodeResults = results;
   }

   // src/store/barcodes/state.js
   // Always use a function to return state if you use SSR
   export default function () {
     return {
       barcodeResults: []
     }
   }

In the scanner page, if QR codes are found, update the state and return to the index page.

   import { createNamespacedHelpers } from 'vuex-composition-helpers';
   const { useMutations } = createNamespacedHelpers('barcodes');
   //...
   setup() {
     const { update } = useMutations(['update']);
     const onScanned = (result:ScanResult) => {
       if (result.results.length>0 && scanned.value == false) {
         scanned.value = true;
         update(result.results);
         router.go(-1);
       }
     };
   }

In the index page, load the results using useState.

   import { createNamespacedHelpers } from 'vuex-composition-helpers';
   const { useState } = createNamespacedHelpers('barcodes');
   setup() {
     const results = useState(["barcodeResults"]);
   }

All right, we've now finished writing the page. We can run the following to test the page:

quasar dev

Use Capacitor to Build Android and iOS Apps

We can take a step further to turn the web app into an Android app or an iOS app using Capacitor. The quasar cli has a Capacitor mode but it uses Capacitor v2. We are going to use Capacitor v4 directly.

Drop Capacitor to the project:

   npm install @capacitor/cli @capacitor/core
   npx cap init

Then, we can create projects for Android and iOS.

   npm install @capacitor/ios @capacitor/android
   npx cap add ios
   npx cap add android

Build the web assets and sync the files to the projects.

   npm run build
   npx cap sync

Use the following commands to start the app:

   npx cap run android // run on Android devices
   npx cap run ios // run on iOS devices

Extra steps are needed.

For the iOS platform:

Add camera permission in Info.plist.

   <key>NSCameraUsageDescription</key>
   <string>For barcode scanning</string>

For the Android platform, we may have to handle the back button event using the @capacitor/app plugin:

   import { App } from '@capacitor/app';
   App.addListener("backButton",() => {
     console.log("back pressed");
   });

For both platforms, enable viewport-fit=cover in the index.template.html so that the page will not be blocked by the status bar.

   - <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width<% if (ctx.mode.cordova || ctx.mode.capacitor) { %>, viewport-fit=cover <% } %>">
   + <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, viewport-fit=cover">

Source Code

https://github.com/tony-xlh/Quasar-QR-Code-Scanner

Transfer Data with Animated QR codes

xulihang — Fri, 08 Oct 2021 10:07:49 +0000

QR code is useful to transfer data across different devices. An Android device can exchange data with an iOS device through a QR code. It does not require setting up a wireless network and does not have the hassle of pairing using Bluetooth. As it is a screen-camera solution, it is also safe to use.

As discussed in the previous article, a QR code can store up to 2,953 bytes with its version set to 40 and its error correction level set to low in bytes mode, which is quite enough for transferring data like a web link, the info of driver's license, etc.

What if we want to transfer more data using QR codes?

We can separate the data into several QR codes and retrieve the entire data by decoding these codes. We can print these codes on one paper or animate them on a screen. The Dynamsoft Barcode Reader (DBR) has the ability to read multiple codes in one image and can also read from the video stream.

In this article, we are going to focus on transferring data with animated QR codes.

There have been studies about this and one of them has stood out which is TXQR. It uses fountain codes to add redundancy to the data so that skipping a few frames will not affect the reading. It uses a repeated codes mode before turning to fountain codes. The program is implemented in Go.

However, using fountain codes brings extra complexity to the process, and actually, with a good barcode reading SDK, the repetition mode can also have a satisfying rate. In the following part, we are going to create an animated QR code generator and an animated QR code reader in pure JavaScript. Users can select a file to transfer and retrieve it on another device by scanning the QR codes.

Animated QR Codes Generator in JavaScript

The file to transfer is first read as bytes and divided into chunks by a fixed chunk size. Metadata is appended to each chunk's beginning. The metadata is simple. It is the index of the chunk and the total number of chunks and if the chunk is the first one, the filename and the mime type are appended as well.

An example of chunks:

"1/11|example.webp|image/webp|bytes"
"2/11|bytes"
"3/11|bytes"

After the chunks are created, we use a JavaScript library to generate QR codes and animate them in a loop.

Users can adjust the chunk size and interval of QR codes to help the receiver have a better reading.

Here are the key parts of the code.

<input type="file" id="file" onchange="loadfile()"/>
<label for="name">Chunk size (bytes):</label>
<input type="text" id="chunkSize" name="chunkSize" value="2000">
<label for="name">Extra interval (ms):</label>
<input type="text" id="interval" name="interval" value="200">
<div id="placeHolder"></div>
<script>
qrcode.stringToBytes = function(data) { return data; }; //store bytes directly
function loadfile() { 
    let files = document.getElementById('file').files;
    if (files.length == 0) {
        return;
    }
    var file = files[0];
    fileReader = new FileReader();
    fileReader.onload = function(e){
        loadArrayBufferToChunks(e.target.result,file.name,file.type);
        showAnimatedQRCode();
    };
    fileReader.onerror = function () {
        console.warn('oops, something went wrong.');
    };
    fileReader.readAsArrayBuffer(file);
}

function loadArrayBufferToChunks(bytes,filename,type){
    var bytes = new Uint8Array(bytes);
    var data = concatTypedArrays(stringToBytes(encodeURIComponent(filename)+"|"+type+"|"),bytes); //The filename is encoded for non-ascii characters like Chinese.
    var chunkSize = parseInt(document.getElementById("chunkSize").value);
    var num = Math.ceil(data.length / chunkSize)
    chunks=[];
    for (var i=0;i<num;i++){
        var start = i*chunkSize;
        var chunk = data.slice(start,start+chunkSize);
        var meta = (i+1)+"/"+num+"|";
        chunk = concatTypedArrays(stringToBytes(meta),chunk);
        chunks.push(chunk);
    }
}

function showAnimatedQRCode(){
    createQRCode(chunks[currentIndex]);
    currentIndex = currentIndex + 1
    var interval = parseInt(document.getElementById("interval").value)
    setTimeout("showAnimatedQRCode()",interval);
}

function createQRCode(data){
    var typeNumber = 0;
    var errorCorrectionLevel = 'L';
    var qr = qrcode(typeNumber, errorCorrectionLevel);
    qr.addData(data);
    qr.make();
    var placeHolder = document.getElementById('placeHolder');
    placeHolder.innerHTML = qr.createSvgTag(); // or createImgTag
}

function stringToBytes(s) {
    var bytes = [];
    for (var i = 0; i < s.length; i += 1) {
        var c = s.charCodeAt(i);
        bytes.push(c & 0xff);
    }
    return bytes;
}

//https://stackoverflow.com/questions/33702838/how-to-append-bytes-multi-bytes-and-buffer-to-arraybuffer-in-javascript
function concatTypedArrays(a, b) { //array + unint8 array
    var newLength = a.length + b.byteLength;
    console.log(newLength);
    var c = new Uint8Array(newLength);
    c.set(a, 0);
    c.set(b, a.length);
    return c;
}
</script>

Animated QR Codes Reader in JavaScript

Now that we have a generator. We need a reader to read the animated QR codes. We can create native mobile applications with high performance using the Dynamsoft Barcode Reader's mobile SDK. But for the ease of use and cross-platform concerns, here, we are gonna use the JavaScript version of DBR to create the reader.

Read QR Codes from Video Stream

The JavaScript version of DBR is easy to use. We can start reading QR codes from the video stream using the following code.

<!DOCTYPE html>
<html>
<head>
    <meta name="viewport" content="width=device-width,initial-scale=1.0,maximum-scale=1.0,user-scalable=0" />
    <script src="https://cdn.jsdelivr.net/npm/dynamsoft-javascript-barcode@8.6.1/dist/dbr.js"></script>
</head>
<body>
<input type="button" value="Start Scanning" onclick="startScanning();" />
<script>
let scanner = null;
init();
async function init(){
    scanner = await Dynamsoft.DBR.BarcodeScanner.createInstance();
    scanner.onFrameRead = results => {
        //handle barcode results
    };
    scanner.onUnduplicatedRead = (txt, result) => {
    };
}

async function startScanning(){
    if (scanner==null){
        alert("Please wait for the initialization of the scanner.")
        return;
    }
    await scanner.show();
}
</script>
</body>
</html>

Read Chunks and Restore the Original Data

Now, we have to parse the data of the QR codes and try to restore the data after all the chunks are received.

var total = 0;
var code_results = {};
function processRead(result){
    var text = result["barcodeText"];
    try {
        var meta = text.split("|")[0];
        var totalOfThisOne = parseInt(meta.split("/")[1]);
        if (total!=0){
            if (total != totalOfThisOne){ //QR codes for another file
                total = totalOfThisOne;
                code_results={};
                return;
            }
        }

        total = totalOfThisOne;
        var index = parseInt(meta.split("/")[0]);
        code_results[index]=result;
        if (getObjectLength(code_results)==total){
            onCompleted();
        }

    } catch(error) {
        console.log(error);
    }
}
function onCompleted(){
    showResult(timeElapsed);
}

async function showResult(timeElapsed){
    //combine chunks of bytes
    var jointData = [];
    for (var i=0;i<getObjectLength(code_results);i++){
        var index = i+1;
        var result = code_results[index];
        var bytes = result.barcodeBytes;
        var text = result.barcodeText;
        if (index == 1){
            var filename = text.split("|")[1]; //the first one contains progress|filename|image/webp|data
            var mimeType = text.split("|")[2];
            var firstSeparatorIndex = text.indexOf("|");
            var secondSeparatorIndex = text.indexOf("|",firstSeparatorIndex+1);
            var dataStart = text.indexOf("|",secondSeparatorIndex+1)+1;
            data = bytes.slice(dataStart,bytes.length);
        }else{
            var dataStart = text.indexOf("|")+1;
            data = bytes.slice(dataStart,bytes.length);
        }
        jointData = jointData.concat(data);
    }
    //display results on the page
    //...
    resetResults();
}

function resetResults(){
    code_results={};
    total = 0;
}

The decoded results are displayed in a list on the web page.

One list item contains a link to download the file, the elapsed time and the download speed. If the file is an image, an img element will be appended as well.

HTML:

<div id="decodedResults">
    Results:
    <ol id="decodedList">
    </ol>
</div>

JavaScript:

async function showResult(timeElapsed){
    //......
    var decodedList = document.getElementById("decodedList");
    var item = document.createElement("li");
    decodedList.appendChild(item);
    var dataURL = await ArraybufferAsDataURL(jointData,mimeType);
    appendDownloadLink(item, dataURL,filename);
    var info = document.createElement("span");
    info.innerText = " elapsed time: " + timeElapsed + "ms" +" speed: "+ (jointData.length/1024/(timeElapsed/1000)).toFixed(2) +"KB/s";
    item.appendChild(info);
    if (dataURL.indexOf("image")!=-1) {
        var img = document.createElement("img");
        img.src = dataURL;
        img.style.display = "block";
        img.style.maxHeight = "350px";
        img.style.maxWidth = "100%";
        item.appendChild(img);
    }
}

//https://stackoverflow.com/questions/12710001/how-to-convert-uint8-array-to-base64-encoded-string
const ArraybufferAsDataURL = async (data,mimeType) => {
    // Use a FileReader to generate a base64 data URI
    const dataUrl = await new Promise((r) => {
        const reader = new FileReader()
        reader.onload = () => r(reader.result)
        var array = ConvertToUInt8Array(data);
        var blob = new Blob([array],{type: mimeType});
        reader.readAsDataURL(blob)
    })
    return dataUrl;
}

function ConvertToUInt8Array(data){
    var array = new Uint8Array(data.length);
    for (var i=0;i<data.length;i++){
        array[i] = data[i];
    }
    return array;
}

function appendDownloadLink(item, dataURL,filename){
    var link = document.createElement('a');
    link.setAttribute('target', '_blank');
    link.setAttribute('href', dataURL);
    if (!filename){
        filename = "file";
    }
    link.setAttribute('download', filename);
    link.innerText=filename;
    item.appendChild(link);
}

function getObjectLength(obj){
    return getObjectKeys(obj).length;
}

function getObjectKeys(obj){
    return Object.keys(obj);
}

During the decoding process, it will also show the statistics like time elapsed, frames processed, successfully-read frames and the progress.

function updateStatistics(timeElapsed){
    var statisticsPre = document.getElementById("statisticsPre");
    statistics = "elapsed time: " + (timeElapsed)/1000 +"s";
    statistics = statistics +"\ntotal frame number: " + framesRead;
    statistics = statistics +"\nsuccessful number: " + successNum;
    statistics = statistics +"\nsuccess fps: " + (successNum/(timeElapsed/1000)).toFixed(2);
    statistics = statistics +"\nprogress: " + getObjectLength(code_results) + "/" + total;
    statisticsPre.innerHTML=statistics;
}

Define the UI

The scanner's UI is customizable.

Create a scanner element and then bind the element with the following code:

await scanner.setUIElement(document.getElementById('scanner'));

There are several pre-defined classes: dbrScanner-video, dbrScanner-cvs-drawarea, dbrScanner-cvs-scanarea, dbrScanner-scanlight, dbrScanner-sel-camera, dbrScanner-sel-resolution. DBR JS will find and make use of them if they exist.

The scanner element:

<div id="scanner" style="display:none;">
    <video class="dbrScanner-video" playsinline="true" style="width:100%;height:100%;position:absolute;left:0;top:0;"></video>
    <canvas class="dbrScanner-cvs-drawarea" style="width:100%;height:100%;position:absolute;left:0;top:0;"></canvas>
    <div class="dbrScanner-cvs-scanarea" style="width:100%;height:100%;position:absolute;left:0;top:0;">
        <div class="dbrScanner-scanlight" style="width:100%;height:3%;position:absolute;animation:3s infinite dbrScanner-scanlight;border-radius:50%;box-shadow:0px 0px 2vw 1px #00e5ff;background:#fff;display:none;"></div>
    </div>
    <select class="dbrScanner-sel-camera" style="margin:0 auto;position:absolute;left:0;top:0;height:30px;"></select>
    <select class="dbrScanner-sel-resolution" style="position:absolute;left:0;top:30px;"></select>
</div>

If you don't want to show some element, like the barcode overlay canvas, you can simply delete it.

Improve the Reading Speed

The transferring speed of this screen-camera solution is mainly decided by how many QR code images can the receiver capture and decode in a fixed period.

Mobile devices can capture 30 frames per second but it may take hundreds of milliseconds to decode one frame. The decoding performance affects the speed the most.

There are several ways to improve this. The Dynamsoft Barcode Reader provides rich parameters to optimize the performance for a specific usage scenario.

Scan only QR Codes

The Dynamsoft Barcode Reader supports a multitude of barcode formats. We can update the settings to scan QR codes only so that it will not spend extra effort finding other barcode formats.

let settings = await scanner.getRuntimeSettings();
settings.barcodeFormatIds = Dynamsoft.DBR.EnumBarcodeFormat.BF_QR_CODE;
await scanner.updateRuntimeSettings(settings);

Set up a scan region

The QR code is just a part of the entire video frame We can set up a scan region so that the QR code takes up most of the frame.

let settings = await scanner.getRuntimeSettings();
settings.region.regionMeasuredByPercentage = 1; //use percentage
var video = document.getElementsByTagName("video")[0];
if (video.videoHeight>video.videoWidth){
    settings.region.regionLeft = 0;
    settings.region.regionTop = 25;
    settings.region.regionRight = 100;
    settings.region.regionBottom = 75;
}else{
    settings.region.regionLeft = 25;
    settings.region.regionTop = 0;
    settings.region.regionRight = 75;
    settings.region.regionBottom = 100;
}
await scanner.updateRuntimeSettings(settings);

Use a faster localization method

The Dynamsoft Barcode Reader comes with several barcodes localization methods. The scan directly mode is suitable for reading QR codes with a phone.

let settings = await scanner.getRuntimeSettings();
settings.localizationModes = [Dynamsoft.DBR.EnumLocalizationMode.LM_SCAN_DIRECTLY, 0, 0, 0, 0, 0, 0, 0, 0];
await scanner.updateRuntimeSettings(settings);

Demo

Here is a demo of the final result running on iOS:

Test and Conclusion

A test is run for this combination of animated QR code generator and scanner to examine how it performs with different chunk sizes and intervals on an iOS device and an Android device. The speed results of three continuous readings are recorded (for large files, only one record).

Here are the test results on iPhone SE 2016:

File size: 15.93KB, Chunk size: 1800, Interval: 100, Speed: 6.90KB/s, 10.68KB/s, 5.35KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 200, Speed: 7.22KB/s, 7.17KB/s, 7.47KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 400, Speed: 3.94KB/s, 3.99KB/s, 3.90KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 800, Speed: 2.09KB/s, 2.08KB/s, 2.05KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 100, Speed: 16.38KB/s, 4.26KB/s, 8.56KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 200, Speed: 11.25KB/s, 11.33KB/s, 11.32KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 400, Speed: 6.78KB/s, 6.74KB/s, 6.74KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 800, Speed: 3.56KB/s, 3.61KB/s, 3.70KB/s
File size: 231KB, Chunk size: 1800, Interval: 400, Speed: 3.01KB/s
File size: 231KB, Chunk size: 2900, Interval: 400, Speed: 2.12KB/s

Here are the test results on Sharp AQUOS S2 (the CPU power is weaker):

File size: 15.93KB, Chunk size: 1800, Interval: 100, Speed: 1.15KB/s, 1.24KB/s, 0.74KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 200, Speed: 2.18KB/s, 1.38KB/s, 1.91KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 400, Speed: 3.44KB/s, 2.60KB/s, 4.06KB/s
File size: 15.93KB, Chunk size: 1800, Interval: 800, Speed: 2.03/KB/s, 2.08KB/s, 2.07KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 100, Speed: 1.05KB/s, 1.80KB/s, 0.93KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 200, Speed: 5.42KB/s, 5.37KB/s, 9.19KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 400, Speed: 5.93KB/s, 7.10KB/s, 3.12KB/s
File size: 15.93KB, Chunk size: 2900, Interval: 800, Speed: 3.90KB/s, 3.53KB/s, 3.68KB/s
File size: 231KB, Chunk size: 1800, Interval: 400, Speed: 2.06KB/s
File size: 231KB, Chunk size: 2900, Interval: 400, Speed: 1.05KB/s

We can make a conclusion based on this test heuristically.

The solution works great for transferring small-sized files which are under 200KB. Because the speed is limited and since more QR codes are needed to encode a large file, the chance of missing frames for large files is high.
The chunk size and interval should be adjusted accordingly. If the chunk size and the interval are small, the generator can generate QR codes fast, however, the receiver may not catch them in time. If the chunk size is large, more data can be transferred in the same time span, but the receiver has to spend more time decoding and it may miss frames, especially for low-end devices.

This solution may be improved in the following ways:

Improve the speed by using a new format, like colored QR code, which can have a larger data capacity.
Improve the missed frames problem by using fountain codes.
Improve the missed frames problem by making the current unidirectional communication to a bidirectional communication so that the generator will only show the undecoded QR codes. This can be done by using the generator to scan a QR code displayed on the receiver's screen.
Use data compression. Instead of transferring JPEG files, transfer WebP files. (conversion tool)

Source Code

https://github.com/xulihang/AnimatedQRCodeReader

Locating and Decoding EAN-13 Barcodes using Python and OpenCV

xulihang — Mon, 13 Sep 2021 01:49:41 +0000

The International Article Number (also known as European Article Number or EAN) is a standard describing a barcode symbology and numbering system used in global trade to identify a specific retail product type. The most commonly used EAN standard is the thirteen-digit EAN-13. ¹

The 13-digit EAN-13 number consists of four components:

GS1 prefix
Manufacturer code
Product code
Check digit

An EAN-13 barcode has 95 areas (also known as modules) of equal width. Each area can be either white (represented here as 0) or black (represented as 1). Continuous areas form a black or white bar. There are 59 bars in an EAN-13 barcode.

From left to right, there are:

3 areas for the start guard (101)
42 left-hand areas (seven per digit) to encode the 2nd to 7th digits. Each digit is represented by four bars. The first digit can be then inferred from the 6 digits.
5 areas for the center guard (01010)
42 right-hand areas (seven per digit) to encode the 8th to 13th digits. Each digit is represented by four bars.
3 areas for the end guard (101)

The encoding of the digit can be known by looking up the following table.

Digit	Left-hand (Odd)	Left-hand (Even)	Right-hand
0	0001101	0100111	1110010
1	0011001	0110011	1100110
2	0010011	0011011	1101100
3	0111101	0100001	1000010
4	0100011	0011101	1011100
5	0110001	0111001	1001110
6	0101111	0000101	1010000
7	0111011	0010001	1000100
8	0110111	0001001	1001000
9	0001011	0010111	1110100

The left-hand digits have a parity property which is odd and even. The initial digit can be inferred by checking the following table.

First digit	The parity of the 6 left-hand digits
0	OOOOOO
1	OOEOEE
2	OOEEOE
3	OOEEEO
4	OEOOEE
5	OEEOOE
6	OEEEOO
7	OEOEOE
8	OEOEEO
9	OEEOEO

In the following part, we will try to decode and detect EAN-13 barcodes using Python and OpenCV.

Decoding EAN-13 Barcodes

Based on the specification of EAN-13, we can create an EAN-13 barcode decoder.

Here is a test barcode images generated with an online tool:

Get the Data Sequence

Since it is a generated image, we can directly read the barcode data from it.

Create a thresholded image.

img = cv2.imread("generated.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, thresh =cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

The value of the white pixels of the thresholded images is 255 and the black pixels 0. We need to invert it and replace 255 with 1 to conform to the 0 and 1 pattern. Only one line of the barcode is needed and here, we use the middle line.
```
thresh = cv2.bitwise_not(thresh)
line = thresh[int(img.shape[0]/2)]
for i in range(len(line)):
    if line[i] == 255:
        line[i] = 1
```

Read the 95 areas and detect the module size. The module size is the length of the smallest bar.

def read_bars(line):
    bars = []
    current_length = 1
    for i in range(len(line)-1):
        if line[i] == line[i+1]:
            current_length = current_length + 1
        else:
            bars.append(current_length * str(line[i]))
            current_length = 1
    #remove quite zone
    bars.pop(0)
    return bars

def detect_module_size(bars):
    size = len(bars[0])
    for bar in bars:
        size = min(len(bar),size)
    return size

module_size = detect_module_size(read_bars(line))

Get the data string.

def array_as_string(array, module_size):
    s = ""
    for value in array:
        s = s + str(value)
    s=s.replace("1"*module_size,"1")
    s=s.replace("0"*module_size,"0")
    print("Data string: " + s)
    return s

data_string = array_as_string(line,module_size)

The data string of the test image:

00000000000101011101100010010100111001001101001110011001010101000010100010011101001010000110110010111001010000000

Decode the Data

Now we can separate the data string by the fixed width of digits and guard markers and decode them according to the encoding table.

Decode the left half.

def decode_left_bar_pattern(pattern):
    left_pattern_dict = {}
    left_pattern_dict["0001101"] = {"code":0,"parity":"O"}
    left_pattern_dict["0100111"] = {"code":0,"parity":"E"}
    left_pattern_dict["0011001"] = {"code":1,"parity":"O"}
    left_pattern_dict["0110011"] = {"code":1,"parity":"E"}
    left_pattern_dict["0010011"] = {"code":2,"parity":"O"}
    left_pattern_dict["0011011"] = {"code":2,"parity":"E"}
    left_pattern_dict["0111101"] = {"code":3,"parity":"O"}
    left_pattern_dict["0100001"] = {"code":3,"parity":"E"}
    left_pattern_dict["0100011"] = {"code":4,"parity":"O"}
    left_pattern_dict["0011101"] = {"code":4,"parity":"E"}
    left_pattern_dict["0110001"] = {"code":5,"parity":"O"}
    left_pattern_dict["0111001"] = {"code":5,"parity":"E"}
    left_pattern_dict["0101111"] = {"code":6,"parity":"O"}
    left_pattern_dict["0000101"] = {"code":6,"parity":"E"}
    left_pattern_dict["0111011"] = {"code":7,"parity":"O"}
    left_pattern_dict["0010001"] = {"code":7,"parity":"E"}
    left_pattern_dict["0110111"] = {"code":8,"parity":"O"}
    left_pattern_dict["0001001"] = {"code":8,"parity":"E"}
    left_pattern_dict["0001011"] = {"code":9,"parity":"O"}
    left_pattern_dict["0010111"] = {"code":9,"parity":"E"}
    return left_pattern_dict[pattern]

guard_pattern = "101"
center_guard_pattern = "01010"

begin_index = data_string.find(guard_pattern)+len(guard_pattern)
data_string_left = data_string[begin_index:-1]

left_codes = []
for i in range(6):
    start_index = i*7
    bar_pattern = data_string_left[start_index:start_index+7]
    decoded = decode_left_bar_pattern(bar_pattern)
    left_codes.append(decoded)

Get the initial digit.

def get_first_digit(left_codes):
    parity_dict = {}
    parity_dict["OOOOOO"] = 0
    parity_dict["OOEOEE"] = 1
    parity_dict["OOEEOE"] = 2
    parity_dict["OOEEEO"] = 3
    parity_dict["OEOOEE"] = 4
    parity_dict["OEEOOE"] = 5
    parity_dict["OEEEOO"] = 6
    parity_dict["OEOEOE"] = 7
    parity_dict["OEOEEO"] = 8
    parity_dict["OEEOEO"] = 9
    parity = ""
    for code in left_codes:
        parity = parity + code["parity"]
    return parity_dict[parity]

Decode the right half.

def decode_right_bar_pattern(pattern):
    right_pattern_dict = {}
    right_pattern_dict["1110010"] = {"code":0}
    right_pattern_dict["1100110"] = {"code":1}
    right_pattern_dict["1101100"] = {"code":2}
    right_pattern_dict["1000010"] = {"code":3}
    right_pattern_dict["1011100"] = {"code":4}
    right_pattern_dict["1001110"] = {"code":5}
    right_pattern_dict["1010000"] = {"code":6}
    right_pattern_dict["1000100"] = {"code":7}
    right_pattern_dict["1001000"] = {"code":8}
    right_pattern_dict["1110100"] = {"code":9}
    return right_pattern_dict[pattern]

center_index = data_string_left.find(center_guard_pattern)+len(center_guard_pattern)
data_string_left = data_string_left[center_index:-1]

right_codes = []
for i in range(6):
    start_index = i*7
    bar_pattern = data_string_left[start_index:start_index+7]
    decoded = decode_right_bar_pattern(bar_pattern)
    right_codes.append(decoded)

Check if the code is valid.

We can calculate the checksum and see if it matches the final digit.

def verify(ean13):
    weight = [1,3,1,3,1,3,1,3,1,3,1,3,1,3]
    weighted_sum = 0
    for i in range(12):
        weighted_sum = weighted_sum + weight[i] * int(ean13[i])
    weighted_sum = str(weighted_sum)
    checksum = 0
    units_digit = int(weighted_sum[-1])
    if units_digit != 0:
        checksum = 10 - units_digit
    else:
        checksum = 0
    print("The checksum of "+ean13 + " is " + str(checksum))
    if checksum == int(ean13[-1]):
        print("The code is valid.")
        return True
    else:
        print("The code is invalid.")
        return False

Make the Decoding More Robust

The above decoding method has high requirements of image qualities. It cannot decode barcodes in the real world.

Here is a scan of a printed EAN-13 barcode and its thresholded images. We can see that the bars have many noises and the width of areas is affected by the printing.

For example, the left guard pattern should be 101 while in the scan, it is 1011, which makes it impossible to correctly detect where the barcode starts.

There are ways to improve the decoding like median blur, smooth and scanning every line. One of the ways which prove effective is using a similar edge distance algorithm to normalize the length of digit areas since they have a fixed width of 7.

You can check out this repo to learn more.

Detecting EAN-13 Barcodes

Let's take a step further to detect barcodes in an image.

A basic detection method based on morphology and contours finding is used in this article.

Here is a sample image from the Artelab Medium Barcode 1D Collection:

We can see that the barcode has parallel lines, white background and quiet zones, making it very different from the rest of the content.

Let's try to detect the barcode.

Resize the image for normalization.

img = cv2.imread("05102009081.jpg")
scale_percent = 640/img.shape[1]       
width = int(img.shape[1] * scale_percent)
height = int(img.shape[0] * scale_percent)
dim = (width, height)
resized = cv2.resize(img, dim, interpolation = cv2.INTER_AREA)

Create a thresholded image.

gray = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)
ret, thresh =cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)

Invert and dilate.

thresh = cv2.bitwise_not(thresh)
kernel = np.ones((3, 20), np.uint8)
thresh = cv2.dilate(thresh, kernel)

Find contours and get the cropped and rotated candidate areas.

def crop_rect(rect, box, img):
    W = rect[1][0]
    H = rect[1][1]
    Xs = [i[0] for i in box]
    Ys = [i[1] for i in box]
    x1 = min(Xs)
    x2 = max(Xs)
    y1 = min(Ys)
    y2 = max(Ys)

    # Center of rectangle in source image
    center = ((x1+x2)/2,(y1+y2)/2)
    # Size of the upright rectangle bounding the rotated rectangle
    size = (x2-x1, y2-y1)
    # Cropped upright rectangle
    cropped = cv2.getRectSubPix(img, size, center)

    angle = rect[2]
    if angle!=90: #need rotation
        if angle>45:
            angle = 0 - (90 - angle)
        else:
            angle = angle
        M = cv2.getRotationMatrix2D((size[0]/2, size[1]/2), angle, 1.0)

        cropped = cv2.warpAffine(cropped, M, size)
        croppedW = H if H > W else W
        croppedH = H if H < W else W
        # Final cropped & rotated rectangle
        croppedRotated = cv2.getRectSubPix(cropped, (int(croppedW),int(croppedH)), (size[0]/2, size[1]/2))
        return croppedRotated
    return cropped

original_sized = cv2.resize(thresh, (img.shape[1],img.shape[0]), interpolation = cv2.INTER_AREA)
contours, hierarchy = cv2.findContours(original_sized,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)    
candidates = []
index = 0
added_index = []
for cnt in contours:
    rect = cv2.minAreaRect(cnt)
    box = cv2.boxPoints(rect) 
    box = np.int0(box)
    cropped = crop_rect(rect,box,img)
    width = cropped.shape[1]
    child_index = hierarchy[0][index][2]
    #the min width of EAN13 is 95 pixel
    if width>95:
        has_overlapped = False
        if child_index in added_index:
            has_overlapped = True
        if has_overlapped == False:
            added_index.append(index)
            candidate = {"cropped": cropped, "rect": rect}
            candidates.append(candidate)
    index = index + 1

We can get the following candidates. We can later send them to decode.

Put Things Together

Now, we can create an EAN-13 reader combining the detecting and decoding parts.

import decode as decoder
import detect as detector
import cv2
import numpy as np

def decode_image(image):
    result_dict = {}
    results = []        

    candidates = detector.detect(image)
    for i in range(len(candidates)):
        candidate = candidates[i]
        cropped = candidate["cropped"]
        rect = candidate["rect"]
        box = cv2.boxPoints(rect) 
        box = np.int0(box)
        ean13, is_valid, thresh = decoder.decode(cropped)
        if is_valid:
            result = {}
            result["barcodeFormat"] = "EAN13"
            result["barcodeText"] = ean13
            result["x1"] = int(box[0][0])
            result["y1"] = int(box[0][1])
            result["x2"] = int(box[1][0])
            result["y2"] = int(box[1][1])
            result["x3"] = int(box[2][0])
            result["y3"] = int(box[2][1])
            result["x4"] = int(box[3][0])
            result["y4"] = int(box[3][1])
            results.append(result)

    result_dict["results"] = results
    return result_dict

if __name__ == "__main__":
    image = cv2.imread("multiple.jpg")
    result_dict = decode_image(image)
    results = result_dict["results"]
    text = "No barcode found"
    if len(results) > 0:
        for result in results:
            if text == "No barcode found":
                text = "Code: "
            ean13 = result["barcodeText"]
            text = text + ean13 + " "
            cv2.line(image,(result["x1"],result["y1"]),(result["x2"],result["y2"]),(0,255,0),3)
            cv2.line(image,(result["x2"],result["y2"]),(result["x3"],result["y3"]),(0,255,0),3)
            cv2.line(image,(result["x3"],result["y3"]),(result["x4"],result["y4"]),(0,255,0),3)
            cv2.line(image,(result["x4"],result["y4"]),(result["x1"],result["y1"]),(0,255,0),3)
    scale_percent = 640/image.shape[1]       
    width = int(image.shape[1] * scale_percent)
    height = int(image.shape[0] * scale_percent)
    dim = (width, height)
    resized = cv2.resize(image, dim, interpolation = cv2.INTER_AREA)
    cv2.putText(resized, text, (5,50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
    cv2.imshow("result", resized);
    cv2.waitKey(0);
    cv2.destroyAllWindows();

Source Code

https://github.com/xulihang/EAN13_Reader

References

https://en.wikipedia.org/wiki/International_Article_Number ↩