小鸟识别APP

小鸟识别软件

本文利用nanodet网络训练一个识别鸟类的模型，然后将其部署到Android上，利用瞄准镜头来代替传统的画矩形框。具体效果如下所示：

视频地址：https://www.bilibili.com/video/BV1pr4y1b7eF/

app下载地址：https://www.pgyer.com/Q7Wr

一 训练模型

1.使用模型

本文使用nanodet训练模型，当然也可以使用yolo系列或者ssd训练网络模型。训练过程比较简单，这里不做过多叙述。数据集图片如下所示：

这里只展示几张数据集的图片，数据集大概一共有2000张左右，本人已经标注好，如有需要可前往自行下载。

2，数据集下载

复制这段内容后打开天翼云盘手机App，操作更方便哦！链接：https://cloud.189.cn/t/NVfiQbaUzqqi（访问码：z1jz）

二 模型部署

首先将训练好的模型转成onnx格式，然后再进一步转成ncnn格式，最后再进行部署。
为了方便调用手机摄像头，使用camerax来使用手机摄像头，既方便又简单。
瞄准镜头图片：

图片是背景透明的，这样才更加逼真。

自定义一个SurfaceVIew控件，将识别的结果传入自定义SurfaceView控件的绘图方法，然后不停的绘制上图瞄准镜头图片，并加以旋转，这样就可以达到动态瞄准的效果。

自定义SurfaceVIew控件的代码如下所示

package com.myapp.mycamera;

import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.PixelFormat;
import android.graphics.Point;
import android.graphics.PorterDuff;
import android.util.AttributeSet;
import android.util.Log;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.SurfaceView;

import androidx.annotation.NonNull;

import java.util.ArrayList;

public class MyView2 extends SurfaceView implements Runnable, SurfaceHolder.Callback {
    


    private SurfaceHolder mHolder; // 用于控制SurfaceView
    private Thread t; // 声明一条线程
    private boolean flag; // 线程运行的标识，用于控制线程
    private Canvas mCanvas; // 声明一张画布
    private Paint p; // 声明一支画笔
    private int x = -1, y = -1, r = 500; // 圆的坐标和半径

    public Bitmap bitmap;
    public float rate=1f;

    public ArrayList<MainActivity.Data> points=new ArrayList<>();

    public MyView2(Context context) {
    
        super(context);
        Log.i("aa","111");
    }

    int ww,hh;

    public MyView2(Context context, AttributeSet attrs) {
    
        super(context, attrs);
        mHolder = getHolder(); // 获得SurfaceHolder对象
        mHolder.addCallback(this); // 为SurfaceView添加状态监听

        //设置背景透明
        setZOrderOnTop(true);
        mHolder.setFormat(PixelFormat.TRANSLUCENT);


        p = new Paint(); // 创建一个画笔对象
        p.setColor(Color.WHITE); // 设置画笔的颜色为白色

        p.setStyle(Paint.Style.STROKE);
        //设置描边宽度
        p.setStrokeWidth(8);

        setFocusable(true); // 设置焦点


    }

    @Override

    public void surfaceCreated(@NonNull SurfaceHolder holder) {
    

        bitmap= BitmapFactory.decodeResource(getResources(),R.drawable.a1);
        t = new Thread(this); // 创建一个线程对象
        flag = true; // 把线程运行的标识设置成true
        t.start(); // 启动线程

    }

    @Override
    public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width, int height) {
    

    }

    @Override
    public void surfaceDestroyed(@NonNull SurfaceHolder holder) {
    
        flag=false;

    }

    @Override
    public void run() {
    
        while (flag){
    
            //[184, 368, 552, 736, 920, 1104, 1288, 1472]

            try {
    
                doDraw();
// long drawEndTime = System.currentTimeMillis();
                // 休眠时间为每帧动画持续时间减去绘制一帧所耗时间
                Thread.sleep(100);
            } catch (InterruptedException e) {
    
                e.printStackTrace();
            }
        }


    }

    private int s=0;
    private float angle=0.f;
    public void doDraw() {
    
        //角度
        angle+=10;


        mCanvas = mHolder.lockCanvas(); // 获得画布对象，开始对画布画画.

        Log.i("aa",bitmap.getWidth()+" "+bitmap.getHeight());
        //mCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
        //mCanvas.drawRGB(0, 0, 0); // 把画布填充为黑色
        //清除canvas画布
        // 方法一：
        mCanvas.drawColor(0,PorterDuff.Mode.CLEAR);

// Matrix matrix=new Matrix();
// // //图像平移
//
// matrix.setTranslate(0,0);
// //matrix.preScale(0.5f,0.5f,bitmap.getWidth()/2,bitmap.getHeight()/2);
 matrix.preScale(0.5f,0.5f);
// //图像旋转角度和旋转中心
// matrix.preRotate(angle,bitmap.getWidth()/2,bitmap.getHeight()/2);
// mCanvas.drawBitmap(bitmap,matrix,null);





        try{
    

            for (MainActivity.Data p:points){
    
                int w=bitmap.getWidth();
                int h=bitmap.getHeight();

                rate=p.L*2f/w;

// rate+=(float) p.L/500f;

                Log.i("aa","pll="+p.L);

                Matrix matrix = new Matrix();
                //图像缩放

// //图像平移
                matrix.setTranslate(p.X-w/2,p.Y-h/2);
                matrix.preScale(rate,rate,w/2,h/2);
                //图像旋转角度和旋转中心
                matrix.preRotate(angle,w/2,h/2);

                mCanvas.drawBitmap(bitmap,matrix,null);

            }

        } catch (Exception e) {
    
            e.printStackTrace();
        }


        //截取图像区域
        //Rect srcRect=new Rect(s*(bitmap.getWidth()/8),0,(s+1)*bitmap.getWidth()/8,bitmap.getHeight());
        //屏幕绘画区域
// bb-=0.01;
        //Rect dstRect=new Rect(500,0,(int)(bitmap.getWidth()/8*bb)+500,(int)(bitmap.getHeight()*bb));

// s+=1;
// if (s==8){
    
// s=0;
// }

        //Log.i("aa",""+bitmap.getWidth()+" "+bitmap.getHeight());
        //mCanvas.rotate(45);//顺时针旋转画布

        // 旋转图片 动作
// Matrix matrix = new Matrix();
// matrix.postScale(0.5f, 0.5f);

        //图像平移
        //matrix.setTranslate(bitmap.getWidth()/2,bitmap.getHeight()/2);
        //图像旋转
// matrix.preRotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);

        // 创建新的图片
// Bitmap resizedBitmap = Bitmap.createBitmap(bitmap, 0, 0,
// bitmap.getWidth(), bitmap.getHeight(), matrix, true);
//
// bb+=5;


// Rect srcRect=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());
// Rect dstcRect=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());


        //mCanvas.rotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);

// mCanvas.drawBitmap(resizedBitmap,srcRect,dstcRect,null);
// mCanvas.drawBitmap(bitmap,matrix,null);



// //图像平移
// matrix.setTranslate(bitmap.getWidth()/2,bitmap.getHeight()/2);
// //图像旋转角度和旋转中心
// matrix.preRotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);
// //图像缩放
// matrix.postScale(1.5f,1.5f);
//
// mCanvas.drawBitmap(bitmap,matrix,null);

// Rect srcRect2=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());
// Rect dstcRect2=new Rect(500,500,bitmap.getWidth()+500,bitmap.getHeight()+500);




        mHolder.unlockCanvasAndPost(mCanvas); // 完成画画，把画布显示在屏幕上
    }

// @Override
// public boolean onTouchEvent(MotionEvent event) {
    
// p.setARGB((int) (Math.random()*255),(int)(Math.random()*255),(int)(Math.random()*255),(int)(Math.random()*255));
//
// if (event.getAction()==MotionEvent.ACTION_DOWN){
    
// x = (int) event.getX(); // 获得屏幕被触摸时对应的X轴坐标
// y = (int) event.getY(); // 获得屏幕被触摸时对应的Y轴坐标
// Point neepoint=new Point(x,y);
// points.add(neepoint);
// }
//
// return false;
// }
}

MainActivity.java代码：

package com.myapp.mycamera;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AlertDialog;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraControl;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.ImageCaptureException;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.extensions.HdrImageCaptureExtender;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import androidx.lifecycle.LifecycleOwner;

import android.Manifest;
import android.annotation.SuppressLint;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.res.Configuration;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.media.Image;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.util.Size;
import android.view.MotionEvent;
import android.view.View;
import android.widget.Button;
import android.widget.ImageView;
import android.widget.TextView;
import android.widget.Toast;

import com.google.common.util.concurrent.ListenableFuture;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;


public class MainActivity extends AppCompatActivity {
    

    // Used to load the 'native-lib' library on application startup.
// static {
    
// System.loadLibrary("native-lib");
// }

    private SSd sSdnet=new SSd();

    private Executor executor = Executors.newSingleThreadExecutor();
    private final String[] REQUIRED_PERMISSIONS = new String[]{
    "android.permission.CAMERA", "android.permission.WRITE_EXTERNAL_STORAGE"};
    private int REQUEST_CODE_PERMISSIONS = 1001;

    private Button btn;
    private Button btn2;
    private MyView2 myView2;
    private int back=1;
    private ImageView img;
    private TextView txt;

    Mypreview mPreviewView;

    private int[] pics= {
    R.drawable.a1,R.drawable.a2,R.drawable.a3};
    private int pic_id=0;
    private Configuration mConfiguration;
    @Override
    protected void onCreate(Bundle savedInstanceState) {
    
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        //创建弹窗
        AlertDialog alertDialog=new AlertDialog.Builder(MainActivity.this).create();
        alertDialog.setCancelable(false);
        alertDialog.setTitle("友情提示");
        alertDialog.setMessage("在应用中心中有更多好玩有趣的app，欢迎前往下载体验。使用过程中如遇到bug可将其发送至邮箱[email protected]");
        alertDialog.setButton(DialogInterface.BUTTON_NEGATIVE, "知道了",
                new DialogInterface.OnClickListener() {
    
                    @Override
                    public void onClick(DialogInterface dialog, int which) {
    
                        Log.i("aa","知道了");
                    }
                });
        alertDialog.setButton(DialogInterface.BUTTON_POSITIVE, "应用中心",
                new DialogInterface.OnClickListener() {
    
                    @Override
                    public void onClick(DialogInterface dialog, int which) {
    
                        Intent intent=new Intent(MainActivity.this,DownLoad.class);
                        startActivity(intent);
                    }
                });
        alertDialog.show();

        boolean init=sSdnet.Init(getAssets());

        Log.i("aa",""+init);

        //获取设置的配置信息
        mConfiguration = this.getResources().getConfiguration();



        mPreviewView = findViewById(R.id.previewView);
        img=findViewById(R.id.image);
// Bitmap bb=BitmapFactory.decodeResource(getResources(),R.drawable.pic);
//
// SSd.Obj[] outs=sSdnet.Detect(bb,false);
// Log.i("aa",outs+"");


        txt=findViewById(R.id.txt);

        if(allPermissionsGranted()){
    
            startCamera(); //start camera if permission has been granted by user
        } else{
    
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS);
        }

        btn=findViewById(R.id.btn);
        btn.setOnClickListener(new View.OnClickListener() {
    
            @Override
            public void onClick(View v) {
    
                if (camea_id==0){
    
                    camea_id=1;
                }else camea_id=0;
                bindPreview(cameraProvider);
            }
        });

        myView2=findViewById(R.id.myview);
        btn2=findViewById(R.id.btn2);
        btn2.setOnClickListener(new View.OnClickListener() {
    
            @Override
            public void onClick(View v) {
    
                pic_id+=1;
                if (pic_id>2){
    
                    pic_id=0;
                }
                Bitmap bitmap2= BitmapFactory.decodeResource(getResources(),pics[pic_id]);
                myView2.bitmap=bitmap2;

            }
        });

    }

    private ProcessCameraProvider cameraProvider;
    private void startCamera() {
    

        final ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(this);

        cameraProviderFuture.addListener(new Runnable() {
    
            @Override
            public void run() {
    
                try {
    
                    cameraProvider = cameraProviderFuture.get();
                    bindPreview(cameraProvider);


                } catch (ExecutionException | InterruptedException e) {
    
                    // No errors need to be handled for this Future.
                    // This should never be reached.
                }
            }
        }, ContextCompat.getMainExecutor(this));
    }

    long t1=0;
    long t2=0;
    private int camea_id=1;
    private Bitmap bmp;
    private CameraControl cameraControl;
    private boolean su;
    private boolean heng;
    void bindPreview(@NonNull ProcessCameraProvider cameraProvider) {
    

        Preview preview = new Preview.Builder()
                .build();

        @SuppressLint("WrongConstant") CameraSelector cameraSelector = new CameraSelector.Builder()
                .requireLensFacing(camea_id)
                .build();

        ImageAnalysis imageAnalysis = new ImageAnalysis.Builder()
                .build();
        //imageAnalysis.setAnalyzer(cameraExecutor, new MyAnalyzer());
        imageAnalysis.setAnalyzer(executor, new ImageAnalysis.Analyzer() {
    
            @Override
            public void analyze(@NonNull ImageProxy image) {
    
                runOnUiThread(() ->{
    

                    //获取当下屏幕状态
                    int ori = mConfiguration.orientation; //获取屏幕方向
                    if (ori == mConfiguration.ORIENTATION_LANDSCAPE) {
    
                        //横屏
                        heng=true;
                        su=false;
                    } else if (ori == mConfiguration.ORIENTATION_PORTRAIT) {
    
                        //竖屏
                        su=true;
                        heng=false;
                    }


                    t1=t2;
                    t2=System.currentTimeMillis();
                    long fps=1000/(t2-t1);
                    txt.setText("FPS:"+fps);

                    int rotationDegrees = image.getImageInfo().getRotationDegrees();
// Log.i("aa","angle1="+rotationDegrees);
                    //旋转角度
                    int rotation = mPreviewView.getDisplay().getRotation();
// Log.i("aa","angle2="+rotation);



                    //yuv图像数据转bitmap
                    ImageProxy.PlaneProxy[] planes = image.getPlanes();

                    //cameraX 获取yuv
                    ByteBuffer yBuffer = planes[0].getBuffer();
                    ByteBuffer uBuffer = planes[1].getBuffer();
                    ByteBuffer vBuffer = planes[2].getBuffer();

                    int ySize = yBuffer.remaining();
                    int uSize = uBuffer.remaining();
                    int vSize = vBuffer.remaining();

                    byte[] nv21 = new byte[ySize + uSize + vSize];

                    yBuffer.get(nv21, 0, ySize);
                    vBuffer.get(nv21, ySize, vSize);
                    uBuffer.get(nv21, ySize + vSize, uSize);
                    //获取yuvImage
                    YuvImage yuvImage = new YuvImage(nv21, ImageFormat.NV21, image.getWidth(), image.getHeight(), null);
                    //输出流
                    ByteArrayOutputStream out = new ByteArrayOutputStream();
                    //压缩写入out
                    yuvImage.compressToJpeg(new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()), 50, out);
                    //转数组
                    byte[] imageBytes = out.toByteArray();
                    //生成bitmap
                    Bitmap bmp = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);

                    //旋转bitmap
                    Bitmap rotateBitmap=null;
                    if (camea_id==1 && su){
    
                        rotateBitmap = rotateBitmap(bmp, 90);
                    }else if(camea_id==0 && su){
    
                        rotateBitmap = rotateBitmap(bmp, 270);
                    }else if(camea_id==1 && heng){
    
                        rotateBitmap=bmp;
                    }else {
    
                        rotateBitmap=rotateBitmap(bmp, 0);
                    }


                    Bitmap bmp2=rotateBitmap.copy(Bitmap.Config.ARGB_8888, true);

                    SSd.Obj[] outcome=sSdnet.Detect(bmp2,false);
                    if(outcome!=null){
    
                        ArrayList<Data> datas=new ArrayList<>();
                        for (int i=0;i<outcome.length;i++){
    
                            Data data=new Data();

                            float x= outcome[i].x*(float)myView2.getWidth();
                            float y= outcome[i].y*(float)myView2.getHeight();
                            float w=outcome[i].w*(float)myView2.getWidth();
                            float h=outcome[i].h*(float)myView2.getHeight();
                            data.X=(int)x;
                            data.Y=(int)y;
                            data.L=(int)((w+h)/2);
                            datas.add(data);
                            myView2.points=datas;

                            Log.i("aa","ww="+x+" "+y+" "+w+" "+h);
                        }

                    }else {
    
                        ArrayList<Data> datas=new ArrayList<>();
                        myView2.points=datas;
                    }


// Canvas canvas = new Canvas( bmp2 );
// Paint paint = new Paint();
// paint.setColor( Color.RED );
// paint.setStrokeWidth( 10 );
// canvas.drawRect( 20,40,200,400,paint );
// img.setImageBitmap(bmp2);
                    //关闭
                    image.close();

                });


            }
        });

        ImageCapture.Builder builder = new ImageCapture.Builder();

        //Vendor-Extensions (The CameraX extensions dependency in build.gradle)
        HdrImageCaptureExtender hdrImageCaptureExtender = HdrImageCaptureExtender.create(builder);

        // Query if extension is available (optional).
        if (hdrImageCaptureExtender.isExtensionAvailable(cameraSelector)) {
    
            // Enable the extension if available.
            hdrImageCaptureExtender.enableExtension(cameraSelector);
        }

        final ImageCapture imageCapture = builder
                .setTargetRotation(this.getWindowManager().getDefaultDisplay().getRotation())
                .build();

        preview.setSurfaceProvider(mPreviewView.createSurfaceProvider());

        try {
    
            cameraProvider.unbindAll();
            Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview, imageAnalysis, imageCapture);
            cameraControl=camera.getCameraControl();
            mPreviewView.cameraControl=cameraControl;
// cameraControl.setLinearZoom(mPreviewView.rate);

        } catch (Exception e) {
    
            e.printStackTrace();
        }


    }

    private Bitmap rotateBitmap(Bitmap origin, float alpha) {
    
        if (origin == null) {
    
            return null;
        }
        int width = origin.getWidth();
        int height = origin.getHeight();
        Matrix matrix = new Matrix();
        matrix.setRotate(alpha);
        if (camea_id==0){
    
            matrix.postScale(-1,1);
        }
        // 围绕原地进行旋转
        Bitmap newBM = Bitmap.createBitmap(origin, 0, 0, width, height, matrix, false);
        if (newBM.equals(origin)) {
    
            return newBM;
        }
        origin.recycle();
        return newBM;
    }

    //获取权限函数
    private boolean allPermissionsGranted(){
    
        for(String permission : REQUIRED_PERMISSIONS){
    
            if(ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED){
    
                return false;
            }
        }
        return true;
    }


    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
    

        if(requestCode == REQUEST_CODE_PERMISSIONS){
    
            if(allPermissionsGranted()){
    
                startCamera();
            } else{
    
                Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show();
                this.finish();
            }
        }
    }

    public class Data{
    
        int X;
        int Y;
        int L;
    }
}

3.项目下载地址：https://github.com/qwhh11/MyCamera