深入OpenCV Android应用开发中文版 – 第七章代码更新

本书中文版购书链接：京东当当

英文原版购书链接：亚马逊《Mastering OpenCV Android Application Programming》

原书所给代码以章为单位，针对的Android版本从API 19到API 21不等，同时使用的OpenCV库版本也有2.4.9和2.4.10两种。本文给出的代码是在原书代码的基础上，针对Android 7.0（API 24）与OpenCV 3.2进行了修改，使某些操作更为合理。

原书的完整代码可以在这里获取：https://www.packtpub.com/lcode_download/22299。

更新后的代码托管在GitHub：https://github.com/johnhany/MOAAP/tree/master/MOAAP-Chp7-r3。

关于在Android Studio上配置OpenCV开发环境的方法，请参考《在Android Studio上进行OpenCV 3.1开发》。

本章介绍如何在Android Studio 2上通过OpenCV使用两种基本的机器学习算法：支持向量机（SVM）和最近邻分类器（KNN），并开发一个简单的数字识别应用。关于这两种算法的原理可以参考《深入OpenCV Android应用开发》第七章。

本文需要用到MNIST数据集。如果官方网页打不开，可以在这里下载。下载好后，把train-images.idx3-ubyte和train-labels.idx1-ubyte两个文件拷贝到手机（或者模拟器）的SD卡根目录（没有单独SD卡的则拷贝到手机内存的根目录）。

开发环境

Windows 10 x64

Android Studio 2.3.3（Gradle 3.3，Android Plugin 2.3.3）

Android 7.0（API 24）

JDK 8u141

OpenCV 3.2.0 Android SDK

代码及简略解释

1. 创建Android Studio项目，包命名为net.johnhany.moaap_chp7。导入OpenCV-android-sdk\sdk\java到项目中，并为app模块加载模块依赖。

2. 将app\src\main\java\net\johnhany\moaap_chp7\MainActivity.java文件修改为如下内容：

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

package net.johnhany.moaap_chp7;

import android.Manifest;

import android.content.pm.PackageManager;

import android.support.annotation.NonNull;

import android.support.v4.app.ActivityCompat;

import android.support.v4.content.ContextCompat;

import android.support.v7.app.AppCompatActivity;

import android.os.Bundle;

import android.util.Log;

import android.view.Menu;

import android.view.MenuItem;

import android.view.MotionEvent;

import android.view.SurfaceView;

import android.view.WindowManager;

import org.opencv.android.BaseLoaderCallback;

import org.opencv.android.CameraBridgeViewBase.CvCameraViewFrame;

import org.opencv.android.LoaderCallbackInterface;

import org.opencv.android.OpenCVLoader;

import org.opencv.core.Core;

import org.opencv.core.Mat;

import org.opencv.core.Point;

import org.opencv.core.Scalar;

import org.opencv.android.CameraBridgeViewBase;

import org.opencv.android.CameraBridgeViewBase.CvCameraViewListener2;

import org.opencv.imgproc.Imgproc;

import static org.opencv.core.Core.FONT_HERSHEY_SIMPLEX;

public class MainActivity extends AppCompatActivity implements CvCameraViewListener2 {

private CameraBridgeViewBase mOpenCvCameraView;

private DigitRecognizer mnist;

static int REQUEST_CAMERA = 0;

static int REQUEST_EXTERNAL_STROAGE = 0;

static boolean camera_granted = false;

static boolean read_external_storage_granted = false;

private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {

@Override

public void onManagerConnected(int status) {

switch (status) {

case LoaderCallbackInterface.SUCCESS:

{

Log.d("OCR", "Start loading MNIST and creating classifier");

mOpenCvCameraView.enableView();

mnist = new DigitRecognizer("train-images.idx3-ubyte", "train-labels.idx1-ubyte");

Log.d("OCR", "Loading MNIST done");

} break;

default:

{

super.onManagerConnected(status);

} break;

}

};

/** Called when the activity is first created. */

@Override

public void onCreate(Bundle savedInstanceState) {

super.onCreate(savedInstanceState);

getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);

setContentView(R.layout.activity_main);

if (ContextCompat.checkSelfPermission(MainActivity.this, Manifest.permission.CAMERA)

!= PackageManager.PERMISSION_GRANTED) {

Log.i("permission", "request CAMERA");

ActivityCompat.requestPermissions(MainActivity.this, new String[]{Manifest.permission.CAMERA}, REQUEST_CAMERA);

}else {

Log.i("permission", "CAMERA already granted");

camera_granted = true;

}

if (ContextCompat.checkSelfPermission(MainActivity.this, Manifest.permission.READ_EXTERNAL_STORAGE)

!= PackageManager.PERMISSION_GRANTED) {

Log.i("permission", "request EXTERNAL_STROAGE");

ActivityCompat.requestPermissions(MainActivity.this, new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, REQUEST_EXTERNAL_STROAGE);

}else {

Log.i("permission", "EXTERNAL_STROAGE already granted");

read_external_storage_granted = true;

}

OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_3_2_0, this, mLoaderCallback);

mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.java_surface_view);

mOpenCvCameraView.setVisibility(SurfaceView.VISIBLE);

mOpenCvCameraView.setCvCameraViewListener(this);

Log.d("OCR", "onCreate done");

}

@Override

public void onRequestPermissionsResult(int requestCode, @NonNull String permissions[], @NonNull int[] grantResults) {

if (requestCode == REQUEST_CAMERA) {

// If request is cancelled, the result arrays are empty.

if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {

// permission was granted

Log.i("permission", "CAMERA granted");

camera_granted = true;

} else {

// permission denied

Log.i("permission", "CAMERA denied");

}

} if (requestCode == REQUEST_EXTERNAL_STROAGE) {

// If request is cancelled, the result arrays are empty.

if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) {

// permission was granted

Log.i("permission", "EXTERNAL_STROAGE granted");

read_external_storage_granted = true;

} else {

// permission denied

Log.i("permission", "EXTERNAL_STROAGE denied");

}

@Override

public void onPause()

{

super.onPause();

if (mOpenCvCameraView != null)

mOpenCvCameraView.disableView();

}

@Override

public void onResume()

{

super.onResume();

}

public void onDestroy() {

super.onDestroy();

if (mOpenCvCameraView != null)

mOpenCvCameraView.disableView();

}

@Override

public boolean onCreateOptionsMenu(Menu menu) {

return true;

}

@Override

public boolean onOptionsItemSelected(MenuItem item) {

return true;

}

public void onCameraViewStarted(int width, int height) {

}

public void onCameraViewStopped() {

}

public Mat onCameraFrame(CvCameraViewFrame inputFrame) {

//Get image size and draw a rectangle on the image for reference

Mat temp = inputFrame.rgba();

Imgproc.rectangle(temp, new Point(temp.cols()/2 - 200, temp.rows() / 2 - 200), new Point(temp.cols() / 2 + 200, temp.rows() / 2 + 200), new Scalar(255,255,255),1);

Mat digit = temp.submat(temp.rows()/2 - 180, temp.rows() / 2 + 180, temp.cols() / 2 - 180, temp.cols() / 2 + 180).clone();

Core.transpose(digit,digit);

int predict_result = mnist.FindMatch(digit);

Imgproc.putText(temp, Integer.toString(predict_result), new Point(50, 150), FONT_HERSHEY_SIMPLEX, 3.0, new Scalar(0, 0, 255), 5);

return temp;

}

public boolean onTouchEvent(MotionEvent event) {

return super.onTouchEvent(event);

}

3. 修改app\src\main\res\layout\activity_main.xml文件：

<?xml version="1.0" encoding="utf-8"?>

<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"

xmlns:tools="http://schemas.android.com/tools"

android:layout_height="fill_parent"

android:layout_width="fill_parent"

xmlns:opencv="http://schemas.android.com/apk/res-auto"

tools:context="net.johnhany.moaap_chp7.MainActivity"

android:id="@+id/ocr">

<org.opencv.android.JavaCameraView

android:layout_width="fill_parent"

android:layout_height="fill_parent"

android:visibility="gone"

android:id="@+id/java_surface_view"

opencv:camera_id="any" />

</LinearLayout>

4. 在app\src\main\java\net\johnhany\moaap_chp7下新建一个DigitRecognizer.java文件（非Activity类），其内容为：

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

package net.johnhany.moaap_chp7;

import android.os.Environment;

import android.util.Log;

import org.opencv.core.Core;

import org.opencv.core.CvType;

import org.opencv.core.Mat;

import org.opencv.core.Size;

import org.opencv.imgproc.Imgproc;

import org.opencv.ml.KNearest;

import org.opencv.ml.SVM;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.IOException;

import java.nio.ByteBuffer;

import static org.opencv.ml.Ml.ROW_SAMPLE;

public class DigitRecognizer {

private String images_path = "";

private String labels_path = "";

private int total_images = 0;

private int width = 0;

private int height = 0;

private byte[] labels_data;

private KNearest knn = null;

private SVM svm = null;

DigitRecognizer(String images, String labels)

{

images_path = images;

labels_path = labels;

try{

ReadMNISTData();

}

catch (IOException e)

{

Log.i("Read error:", "" + e.getMessage());

}

private void ReadMNISTData() throws FileNotFoundException {

File external_storage = Environment.getExternalStorageDirectory(); //Get SD Card's path

//Read images

Mat training_images;

File mnist_images_file = new File(external_storage, images_path);

FileInputStream images_reader = new FileInputStream(mnist_images_file);

try{

//Read the file headers which contain the total number of images and dimensions. First 16 bytes hold the header

//byte 0 -3 : Magic Number (Not to be used)

//byte 4 - 7: Total number of images in the dataset

//byte 8 - 11: width of each image in the dataset

//byte 12 - 15: height of each image in the dataset

byte [] header = new byte[16];

images_reader.read(header, 0, 16);

//Combining the bytes to form an integer

ByteBuffer temp = ByteBuffer.wrap(header, 4, 12);

total_images = temp.getInt();

width = temp.getInt();

height = temp.getInt();

//Total number of pixels in each image

int px_count = width * height;

training_images = new Mat(total_images, px_count, CvType.CV_8U);

//images_data = new byte[total_images][px_count];

//Read each image and store it in an array.

for (int i = 0 ; i < total_images ; i++)

{

byte[] image = new byte[px_count];

images_reader.read(image, 0, px_count);

training_images.put(i,0,image);

}

training_images.convertTo(training_images, CvType.CV_32FC1);

images_reader.close();

}

catch (IOException e)

{

Log.i("MNIST Read Error:", "" + e.getMessage());

}

//Read Labels

Mat training_labels;

labels_data = new byte[total_images];

File mnist_labels_file = new File(external_storage, labels_path);

FileInputStream labels_reader = new FileInputStream(mnist_labels_file);

try{

training_labels = new Mat(total_images, 1, CvType.CV_8U);

Mat temp_labels = new Mat(1, total_images, CvType.CV_8U);

byte[] header = new byte[8];

//Read the header

labels_reader.read(header, 0, 8);

//Read all the labels at once

labels_reader.read(labels_data,0,total_images);

temp_labels.put(0,0, labels_data);

//Take a transpose of the image

Core.transpose(temp_labels, training_labels);

training_labels.convertTo(training_labels, CvType.CV_32S);

labels_reader.close();

}

catch (IOException e)

{

Log.i("MNIST Read Error:", "" + e.getMessage());

}

//K-NN Classifier

//knn = KNearest.create();

//knn.setDefaultK(10);

//knn.train(training_images, ROW_SAMPLE, training_labels);

//SVM Classifier

//svm = SVM.create();

//svm.train(training_images, ROW_SAMPLE, training_labels);

File mnist_svm_file = new File(external_storage, "SVM_MNIST.xml");

svm = SVM.load(mnist_svm_file.getAbsolutePath());

Log.d("MNIST SVM:", "var_count: " + svm.getVarCount());

}

int FindMatch(Mat test_image)

{

//Dilate the image

Imgproc.dilate(test_image, test_image, Imgproc.getStructuringElement(Imgproc.CV_SHAPE_CROSS, new Size(3,3)));

//Resize the image to match it with the sample image size

Imgproc.resize(test_image, test_image, new Size(width, height));

//Convert the image to grayscale

Imgproc.cvtColor(test_image, test_image, Imgproc.COLOR_RGB2GRAY);

//Adaptive Threshold

Imgproc.adaptiveThreshold(test_image,test_image,255,Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY_INV,15, 2);

Mat test = new Mat(1, test_image.rows() * test_image.cols(), CvType.CV_32FC1);

int count = 0;

for(int i = 0 ; i < test_image.rows(); i++)

{

for(int j = 0 ; j < test_image.cols(); j++) {

test.put(0, count, test_image.get(i, j)[0]);

count++;

}

Mat results = new Mat(1, 1, CvType.CV_8U);

//K-NN Prediction

//return (int)knn.findNearest(test, 10, results);

//SVM Prediction

return (int)svm.predict(test);

}

在这个DigitRecognizer包含了大部分核心代码。如果想使用SVM，注释掉134-135行，并取消132-133行的注释，这样运行时就要先读入训练数据并训练SVM，然后从摄像头读入图像，并对图像中央区域的数字进行分类。由于SVM训练过程计算量比较大，特别是移动平台上缺少多线程等优化时性能更是直线下降。推荐不更改代码，直接读取我训练好的SVM分类器。预训练的分类器文件在这里下载：http://pan.baidu.com/s/1nuS3M9F，下载好后拷贝到手机SD卡根目录。如果想要自己训练SVM，推荐在电脑上进行训练，然后把训练好的分类器文件提供给Android应用使用。训练SVM的C++代码在这里。在Windows平台式配置OpenCV开发环境的方法可以参考《Windows7+VS2012下64位OpenCV3.0+CUDA7.5的编译和部署》。我实验的结果是，迭代训练500次，精度74%，远不能达到实际运用的要求，至少比胡乱猜准一些-_-

如果想要使用KNN，注释掉134-135行、169行，取消掉127-129行、166行的注释。与SVM正相反，KNN的“训练”过程非常快，分类过程反而计算量非常大，因为OpenCV的实现只是把所有样本保留下来，然后通过所有已知样本与新样本计算相似度，在实机上测试时能明显感到卡顿。不过让人惊喜的是，KNN的测试精度能够达到96.65%（C++版本代码在这里），比某些缺乏优化的CNN模型效果还要好。

5. 修改app\src\main\res\AndroidManifest.xml文件为如下内容：

<?xml version="1.0" encoding="utf-8"?>

<manifest xmlns:android="http://schemas.android.com/apk/res/android"

package="net.johnhany.moaap_chp7">

<supports-screens android:resizeable="true"

android:smallScreens="true"

android:normalScreens="true"

android:largeScreens="true"

android:anyDensity="true" />

<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />

<uses-permission android:name="android.permission.CAMERA"/>

<uses-feature android:name="android.hardware.camera" android:required="false"/>

<uses-feature android:name="android.hardware.camera.autofocus" android:required="false"/>

<uses-feature android:name="android.hardware.camera.front" android:required="false"/>

<uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false"/>

<application

android:allowBackup="true"

android:icon="@mipmap/ic_launcher"

android:label="@string/app_name"

android:supportsRtl="true"

android:theme="@style/AppTheme">

<activity android:name=".MainActivity"

android:screenOrientation="landscape">

<intent-filter>

</intent-filter>

</activity>

</application>

</manifest>

6. 修改app\src\main\res\values\strings.xml文件：

<string name="app_name">第七章 - 深入OpenCV Android应用开发</string>

</resources>

7. 修改app\src\main\res\values\styles.xml文件：

<item name="android:windowActionBar">false</item>

<item name="colorPrimary">@color/colorPrimary</item>

<item name="colorPrimaryDark">@color/colorPrimaryDark</item>

<item name="colorAccent">@color/colorAccent</item>

</style>

</resources>

8. app\build.gradle文件修改为：

apply plugin: 'com.android.application'

android {

compileSdkVersion 24

buildToolsVersion "26.0.1"

defaultConfig {

applicationId "net.johnhany.moaap_chp7"

minSdkVersion 16

targetSdkVersion 24

versionCode 1

versionName "1.0"

testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"

}

buildTypes {

release {

minifyEnabled false

proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'

}

dependencies {

compile fileTree(include: ['*.jar'], dir: 'libs')

androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', {

exclude group: 'com.android.support', module: 'support-annotations'

})

compile 'com.android.support:appcompat-v7:24.2.1'

compile 'com.android.support.constraint:constraint-layout:1.0.2'

testCompile 'junit:junit:4.12'

compile project(':openCVLibrary320')

}

9. openCVLibrary320\build.gradle文件修改为：

apply plugin: 'com.android.library'

android {

compileSdkVersion 24

buildToolsVersion "26.0.1"

defaultConfig {

minSdkVersion 16

targetSdkVersion 24

}

buildTypes {

release {

minifyEnabled false

proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt'

}

10. 检查一下项目根目录的build.gradle文件是否为如下内容：

// Top-level build file where you can add configuration options common to all sub-projects/modules.

buildscript {

repositories {

jcenter()

}

dependencies {

classpath 'com.android.tools.build:gradle:2.3.3'

// NOTE: Do not place your application dependencies here; they belong

// in the individual module build.gradle files

}

allprojects {

repositories {

jcenter()

}

task clean(type: Delete) {

delete rootProject.buildDir

}

运行效果

SVM运行效果：

KNN运行效果：

参考

http://answers.opencv.org/question/93682/svm-model-fails-to-load/?answer=93683#post-id-93683

JohnHany的博客

编程手绘文字

深入OpenCV Android应用开发中文版 – 第七章代码更新

8