英文原版链接:亚马逊《Mastering OpenCV Android Application Programming》
原书所给代码以章为单位,针对的Android版本从API 19到API 21不等,同时使用的OpenCV库版本也有2.4.9和2.4.10两种。本文给出的代码是在原书代码的基础上,针对Android 7.0(API 24)与OpenCV 3.2进行了修改,适当地修改了一些操作,以使代码整体上更合理。
原书的完整代码可以在这里获取:https://www.packtpub.com/lcode_download/22299。
更新后的代码托管在GitHub:https://github.com/johnhany/MOAAP/tree/master/MOAAP-Chp6-r3。
关于在Android Studio上配置OpenCV开发环境的方法,请参考《在Android Studio上进行OpenCV 3.1开发》。
关于在Android Studio上利用CMAKE配置OpenCV NDK开发环境的方法,请参考《Android Studio 2.3利用CMAKE进行OpenCV 3.2的NDK开发》。
本章介绍如何在Android Studio 2上利用OpenCV 3.2的Java API和Native API,通过Android NDK r15b开发一个图像拼接应用,涉及特征检测与匹配、图像匹配、光束平差法、自动校直、增益补偿、多频段融合等知识。但我们的应用只需要在C++部分调用OpenCV的Stitcher就可完成整个拼接过程。关于以上各算法的原理可以参考《深入OpenCV Android应用开发》第六章。
开发环境
Windows 10 x64专业版
Android Studio 2.3.3(Gradle 3.3,Android Plugin 2.3.3)
Android 7.0(API 24)
Android NDK r15b
JDK 8u141
OpenCV 3.2.0 Android SDK
代码及简略解释
1. 创建Android Studio项目,包命名为net.johnhany.moaap_chp6。注意我们需要建立一个NDK项目,所以需要勾选Include C++ Support。导入OpenCV-android-sdk\sdk\java到项目中,并为app模块加载模块依赖。
2. 在app\src\main\java目录中找到net.johnhany.moaap_chp6包,为MainActivity.java添加如下代码:
package net.johnhany.moaap_chp6; import android.Manifest; import android.app.ProgressDialog; import android.content.Intent; import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.graphics.BitmapFactory; import android.net.Uri; import android.os.AsyncTask; import android.os.Bundle; import android.os.Environment; import android.provider.MediaStore; import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; import android.support.v7.app.AppCompatActivity; import android.util.Log; import android.view.Menu; import android.view.MenuItem; import android.view.View; import android.widget.Button; import android.widget.ImageView; import android.widget.Toast; import org.opencv.android.BaseLoaderCallback; import org.opencv.android.LoaderCallbackInterface; import org.opencv.android.OpenCVLoader; import org.opencv.android.Utils; import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; import java.io.File; import java.io.FileNotFoundException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; public class MainActivity extends AppCompatActivity { private final int CLICK_PHOTO = 1; private final int SELECT_PHOTO = 2; private Uri fileUri; private ImageView ivImage; private Mat src; private List<Mat> listImage = new ArrayList<>(); private static final String FILE_LOCATION = Environment.getExternalStorageDirectory() + "/Download/MOAAP/Chapter6/"; static int REQUEST_READ_EXTERNAL_STORAGE = 11; static boolean read_external_storage_granted = false; private BaseLoaderCallback mOpenCVCallBack = new BaseLoaderCallback(this) { @Override public void onManagerConnected(int status) { switch (status) { case LoaderCallbackInterface.SUCCESS: System.loadLibrary("stitcher"); //DO YOUR WORK/STUFF HERE break; default: super.onManagerConnected(status); break; } } }; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); ivImage = (ImageView)findViewById(R.id.ivImage); OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_3_2_0, this, mOpenCVCallBack); if (ContextCompat.checkSelfPermission(MainActivity.this, Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { Log.i("permission", "request READ_EXTERNAL_STORAGE"); ActivityCompat.requestPermissions(MainActivity.this, new String[]{Manifest.permission.READ_EXTERNAL_STORAGE}, REQUEST_READ_EXTERNAL_STORAGE); }else { Log.i("permission", "READ_EXTERNAL_STORAGE already granted"); read_external_storage_granted = true; } Button bClickImage, bDone; bClickImage = (Button)findViewById(R.id.bClickImage); bDone = (Button)findViewById(R.id.bDone); bClickImage.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE); File imagesFolder = new File(FILE_LOCATION); //imagesFolder.mkdirs(); File image = new File(imagesFolder, "panorama_"+ (listImage.size()+1) + ".jpg"); fileUri = Uri.fromFile(image); Log.d("MainActivity", "File URI = " + fileUri.toString()); intent.putExtra(MediaStore.EXTRA_OUTPUT, fileUri); // set the image file name // start the image capture Intent startActivityForResult(intent, CLICK_PHOTO); } }); bDone.setOnClickListener(new View.OnClickListener() { @Override public void onClick(View v) { if(listImage.size()==0){ Toast.makeText(getApplicationContext(), "No images clicked", Toast.LENGTH_SHORT).show(); } else if(listImage.size()==1){ Toast.makeText(getApplicationContext(), "Only one image clicked", Toast.LENGTH_SHORT).show(); Bitmap image = Bitmap.createBitmap(src.cols(), src.rows(), Bitmap.Config.ARGB_8888); Utils.matToBitmap(src, image); ivImage.setImageBitmap(image); } else { createPanorama(); } } }); } @Override public boolean onCreateOptionsMenu(Menu menu) { getMenuInflater().inflate(R.menu.menu_main, menu); return true; } @Override public boolean onOptionsItemSelected(MenuItem item) { int id = item.getItemId(); if (id == R.id.action_load_image && read_external_storage_granted) { Intent photoPickerIntent = new Intent(Intent.ACTION_PICK); photoPickerIntent.setType("image/*"); startActivityForResult(photoPickerIntent, SELECT_PHOTO); return true; } else if(!read_external_storage_granted) { Log.e("MainActivity", "pick image failed"); return true; } return super.onOptionsItemSelected(item); } @Override protected void onActivityResult(int requestCode, int resultCode, Intent imageReturnedIntent) { super.onActivityResult(requestCode, resultCode, imageReturnedIntent); Log.d("MainActivity", "request code " + requestCode + ", click photo " + CLICK_PHOTO + ", result code " + resultCode + ", result ok " + RESULT_OK); switch(requestCode) { case CLICK_PHOTO: if(resultCode == RESULT_OK){ try { Log.d("MainActivity", fileUri.toString()); final InputStream imageStream = getContentResolver().openInputStream(fileUri); final Bitmap selectedImage = BitmapFactory.decodeStream(imageStream); src = new Mat(selectedImage.getHeight(), selectedImage.getWidth(), CvType.CV_8UC4); Imgproc.resize(src, src, new Size(src.rows()/4, src.cols()/4)); Utils.bitmapToMat(selectedImage, src); Imgproc.cvtColor(src, src, Imgproc.COLOR_BGR2RGB); listImage.add(src); } catch (FileNotFoundException e) { e.printStackTrace(); } } break; case SELECT_PHOTO: if(resultCode == RESULT_OK && read_external_storage_granted){ try { final Uri imageUri = imageReturnedIntent.getData(); final InputStream imageStream = getContentResolver().openInputStream(imageUri); final Bitmap selectedImage = BitmapFactory.decodeStream(imageStream); src = new Mat(selectedImage.getHeight(), selectedImage.getWidth(), CvType.CV_8UC4); Imgproc.resize(src, src, new Size(src.rows()/4, src.cols()/4)); Utils.bitmapToMat(selectedImage, src); Imgproc.cvtColor(src, src, Imgproc.COLOR_RGBA2BGR); Log.d("MainActivity", "image height " + src.rows() + ", image width " + src.cols()); listImage.add(src); } catch (FileNotFoundException e) { e.printStackTrace(); } } break; } } @Override public void onRequestPermissionsResult(int requestCode, @NonNull String permissions[], @NonNull int[] grantResults) { if (requestCode == REQUEST_READ_EXTERNAL_STORAGE) { // If request is cancelled, the result arrays are empty. if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) { // permission was granted Log.i("permission", "READ_EXTERNAL_STORAGE granted"); read_external_storage_granted = true; } else { // permission denied Log.i("permission", "READ_EXTERNAL_STORAGE denied"); } } } private void createPanorama(){ new AsyncTask<Void, Void, Bitmap>() { ProgressDialog dialog; @Override protected void onPreExecute() { super.onPreExecute(); dialog = ProgressDialog.show(MainActivity.this, "Building Panorama", "Please Wait"); } @Override protected Bitmap doInBackground(Void... params) { int elems = listImage.size(); long[] tempobjadr = new long[elems]; for (int i=0; i<elems; i++){ tempobjadr[i] = listImage.get(i).getNativeObjAddr(); } Mat result = new Mat(); int stitchstatus = StitchPanorama(tempobjadr, result.getNativeObjAddr()); Log.d("MainActivity", "result height " + result.rows() + ", result width " + result.cols()); if(stitchstatus != 0){ Log.e("MainActivity", "Stitching failed: " + stitchstatus); return null; } Imgproc.cvtColor(result, result, Imgproc.COLOR_BGR2RGBA); Bitmap bitmap = Bitmap.createBitmap(result.cols(), result.rows(), Bitmap.Config.ARGB_8888); Utils.matToBitmap(result, bitmap); return bitmap; } @Override protected void onPostExecute(Bitmap bitmap) { super.onPostExecute(bitmap); dialog.dismiss(); if(bitmap!=null) { ivImage.setImageBitmap(bitmap); } } }.execute(); } @Override protected void onResume() { super.onResume(); } public native int StitchPanorama(long[] imageAddressArray, long outputAddress); }
原书给出的代码是保留一个空的MainActivity,创建一个StitchingActivity,然后所有的操作在StitchingActivity当中执行。我这里改为全部Java操作只在MainActivity中执行。另外,Java层与Native层之间图像数据的传递方式也产生了一些变化。而且,原书所给代码只提供了一个用于载入图像的菜单按钮,并没有实际提供对从内存中读取的图像进行拼接的功能。我修改为即可以通过摄像头拍照进行拼接,也可以通过读取已有图像的方式来拼接。
3. 修改app\src\main\res\layout\activity_main.xml内容如下:
<?xml version="1.0" encoding="utf-8"?> <ScrollView xmlns:android="http://schemas.android.com/apk/res/android" xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" tools:context="net.johnhany.moaap_chp6.MainActivity"> <LinearLayout android:orientation="vertical" android:layout_width="match_parent" android:layout_height="wrap_content"> <ImageView android:layout_width="match_parent" android:layout_height="0dp" android:layout_weight="0.5" android:id="@+id/ivImage" /> <LinearLayout android:layout_width="match_parent" android:layout_height="wrap_content" android:orientation="horizontal"> <Button android:layout_width="match_parent" android:layout_height="wrap_content" android:layout_weight="0.5" android:id="@+id/bClickImage" android:text="Click more images"/> <Button android:layout_width="match_parent" android:layout_height="wrap_content" android:layout_weight="0.5" android:id="@+id/bDone" android:text="Done"/> </LinearLayout> </LinearLayout> </ScrollView>
4. 修改app/CMakeLists.txt内容如下:
cmake_minimum_required(VERSION 3.4.1) set(CMAKE_VERBOSE_MAKEFILE on) set(ocvlibs "E:/dev-lib/opencv-contrib-android-sdk/sdk/native/libs") include_directories(E:/dev-lib/opencv-contrib-android-sdk/sdk/native/jni/include) add_library(libopencv_java3 SHARED IMPORTED ) set_target_properties(libopencv_java3 PROPERTIES IMPORTED_LOCATION "${ocvlibs}/${ANDROID_ABI}/libopencv_java3.so") add_library( # Sets the name of the library. stitcher # Sets the library as a shared library. SHARED # Provides a relative path to your source file(s). src/main/cpp/stitcher.cpp ) find_library( # Sets the name of the path variable. log-lib # Specifies the name of the NDK library that # you want CMake to locate. log ) target_link_libraries( # Specifies the target library. stitcher android log libopencv_java3 # Links the target library to the log library # included in the NDK. ${log-lib} )
关于该文件中各项的含义,请参考《Android Studio 2.3利用CMAKE进行OpenCV 3.2的NDK开发》。另外,我这里引用的是包含opencv_contrib模块的OpenCV Android SDK,其实直接引用官方预编译SDK也可。
5. 将项目默认产生的app\src\main\cpp\native-lib.cpp文件更名为stitcher.cpp,然后将文件内容修改为:
#include <jni.h> #include <vector> #include <android/log.h> #include "opencv2/opencv.hpp" using namespace cv; using namespace std; char filepath1[100] = "/storage/emulated/0/Download/MOAAP/Chapter6/panorama_stitched.jpg"; #define LOG_TAG "MOAAP-CHP6" #define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__) #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) extern "C" { JNIEXPORT jint JNICALL Java_net_johnhany_moaap_1chp6_MainActivity_StitchPanorama(JNIEnv *env, jobject, jlongArray imageAddressArray, jlong outputAddress) { jsize a_len = env->GetArrayLength(imageAddressArray); jlong *imgAddressArr = env->GetLongArrayElements(imageAddressArray,0); vector<Mat> imgVec; for(int k=0; k<a_len; k++) { Mat &curimage = *(Mat*)imgAddressArr[k]; Mat newimage; curimage.copyTo(newimage); float scale = 500.0f / curimage.rows; resize(newimage, newimage, Size((int)(scale*curimage.cols), (int)(scale*curimage.rows))); LOGD("Image height %d width %d", newimage.rows, newimage.cols); imgVec.push_back(newimage); } Mat &result = *(Mat*)outputAddress; Stitcher stitcher = Stitcher::createDefault(); Stitcher::Status status = stitcher.stitch(imgVec, result); LOGD("Result height %d width %d", result.rows, result.cols); imwrite(filepath1, result); return status; } }
6. 在app\src\main\res目录下创建一个名为menu的Andorid resource directory,再在res\menu中创建一个menu_main.xml文件:
<?xml version="1.0" encoding="utf-8"?> <menu xmlns:android="http://schemas.android.com/apk/res/android" xmlns:tools="http://schemas.android.com/tools" xmlns:app="http://schemas.android.com/apk/res-auto"> <item android:id="@+id/action_load_image" android:title="@string/action_load_image" android:orderInCategory="1" app:showAsAction="never" /> </menu>
7. 修改app\src\main\res\values\strings.xml文件:
<resources> <string name="app_name">第六章 - 深入OpenCV Android应用开发</string> <string name="action_load_image">Load Image</string> <string name="title_activity_stitching">第六章 - 深入OpenCV Android应用开发</string> <string name="title_activity_home">第六章 - 深入OpenCV Android应用开发</string> </resources>
8. 修改app\src\main\res\AndroidManifest.xml文件为如下内容:
<?xml version="1.0" encoding="utf-8"?> <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="net.johnhany.moaap_chp6"> <supports-screens android:anyDensity="true" android:largeScreens="true" android:normalScreens="true" android:resizeable="true" android:smallScreens="true" /> <uses-permission android:name="android.permission.CAMERA" /> <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" /> <uses-feature android:name="android.hardware.camera" android:required="false" /> <uses-feature android:name="android.hardware.camera.autofocus" android:required="false" /> <uses-feature android:name="android.hardware.camera.front" android:required="false" /> <uses-feature android:name="android.hardware.camera.front.autofocus" android:required="false" /> <application android:allowBackup="true" android:icon="@mipmap/ic_launcher" android:label="@string/app_name" android:supportsRtl="true"> <activity android:name=".MainActivity" android:label="@string/title_activity_stitching" android:theme="@style/AppTheme"> <intent-filter> <action android:name="android.intent.action.MAIN" /> <category android:name="android.intent.category.LAUNCHER" /> </intent-filter> </activity> </application> </manifest>
9. 将app\build.gradle文件修改为:
apply plugin: 'com.android.application' android { compileSdkVersion 24 buildToolsVersion "26.0.1" defaultConfig { applicationId "net.johnhany.moaap_chp6" minSdkVersion 16 targetSdkVersion 24 versionCode 1 versionName "1.0" testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" externalNativeBuild { cmake { cppFlags "-std=c++11", "-frtti", "-fexceptions" abiFilters 'x86', 'x86_64', 'armeabi', 'armeabi-v7a', 'arm64-v8a', 'mips', 'mips64' } } } sourceSets { main { jniLibs.srcDirs = ['E:\\dev-lib\\opencv-contrib-android-sdk\\sdk\\native\\libs'] } } buildTypes { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } } externalNativeBuild { cmake { path "CMakeLists.txt" } } } dependencies { compile fileTree(include: ['*.jar'], dir: 'libs') androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { exclude group: 'com.android.support', module: 'support-annotations' }) compile 'com.android.support:appcompat-v7:24.2.1' compile 'com.android.support.constraint:constraint-layout:1.0.2' testCompile 'junit:junit:4.12' compile project(':openCVLibrary320') }
同样地,这里引用的是包含opencv_contrib模块的OpenCV Android SDK,读者可以直接引用官方预编译SDK。
10. openCVLibrary320\build.gradle文件修改为:
apply plugin: 'com.android.library' android { compileSdkVersion 24 buildToolsVersion "26.0.1" defaultConfig { minSdkVersion 16 targetSdkVersion 24 } buildTypes { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } }
11. 检查一下项目根目录的build.gradle文件是否为如下内容:
// Top-level build file where you can add configuration options common to all sub-projects/modules. buildscript { repositories { jcenter() } dependencies { classpath 'com.android.tools.build:gradle:2.3.3' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } } allprojects { repositories { jcenter() } } task clean(type: Delete) { delete rootProject.buildDir }
运行效果
测试图片来自https://github.com/opencv/opencv_extra/tree/master/testdata/stitching。
原始图片:
拼接后的效果: