Bird recognition app

Bird recognition software

This paper makes use of nanodet The network trains a model for identifying birds , Then deploy it to Android On , Use the aiming lens to replace the traditional rectangular frame . The specific effect is shown below ：

Video address ：https://www.bilibili.com/video/BV1pr4y1b7eF/

app Download address ：https://www.pgyer.com/Q7Wr

One Training models

1. Using the model

This article USES the nanodet Training models , You can also use it yolo Series or ssd Training network models . The training process is relatively simple , I don't go into too much narration here . The dataset image is shown below ：

Here are only a few pictures of the dataset , The data set probably has 2000 Zhang Zuo , I have marked , If necessary, you can download it by yourself .

2, Dataset Download

After copying this paragraph, open Tianyi cloud disk mobile phone App, It is more convenient to operate ！ link ：https://cloud.189.cn/t/NVfiQbaUzqqi（ Access code ：z1jz）

Two Model deployment

First, turn the trained model into onnx Format , Then it will be further converted to ncnn Format , Finally, deploy .
For the convenience of calling the mobile camera , Use camerax To use the mobile camera , It's convenient and simple .
Aim at the lens picture ：

The picture has a transparent background , It's more realistic .

Customize a SurfaceVIew Control , Pass the recognized results into the custom SurfaceView Control drawing method , Then keep drawing the above aiming lens picture , And rotate it , In this way, the effect of dynamic aiming can be achieved .

Customize SurfaceVIew The code of the control is as follows

package com.myapp.mycamera;

import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.PixelFormat;
import android.graphics.Point;
import android.graphics.PorterDuff;
import android.util.AttributeSet;
import android.util.Log;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.SurfaceView;

import androidx.annotation.NonNull;

import java.util.ArrayList;

public class MyView2 extends SurfaceView implements Runnable, SurfaceHolder.Callback {
    


    private SurfaceHolder mHolder; //  Used to control the SurfaceView
    private Thread t; //  Declare a thread 
    private boolean flag; //  ID of the thread running , Used to control threads 
    private Canvas mCanvas; //  Declare a canvas 
    private Paint p; //  Declare a brush 
    private int x = -1, y = -1, r = 500; //  The coordinates and radius of the circle 

    public Bitmap bitmap;
    public float rate=1f;

    public ArrayList<MainActivity.Data> points=new ArrayList<>();

    public MyView2(Context context) {
    
        super(context);
        Log.i("aa","111");
    }

    int ww,hh;

    public MyView2(Context context, AttributeSet attrs) {
    
        super(context, attrs);
        mHolder = getHolder(); //  get SurfaceHolder object 
        mHolder.addCallback(this); //  by SurfaceView Add status monitoring 

        // Set background transparency 
        setZOrderOnTop(true);
        mHolder.setFormat(PixelFormat.TRANSLUCENT);


        p = new Paint(); //  Create a brush object 
        p.setColor(Color.WHITE); //  Set the color of the brush to white 

        p.setStyle(Paint.Style.STROKE);
        // Set the stroke width 
        p.setStrokeWidth(8);

        setFocusable(true); //  set focus 


    }

    @Override

    public void surfaceCreated(@NonNull SurfaceHolder holder) {
    

        bitmap= BitmapFactory.decodeResource(getResources(),R.drawable.a1);
        t = new Thread(this); //  Create a thread object 
        flag = true; //  Set the thread running ID to true
        t.start(); //  Start thread 

    }

    @Override
    public void surfaceChanged(@NonNull SurfaceHolder holder, int format, int width, int height) {
    

    }

    @Override
    public void surfaceDestroyed(@NonNull SurfaceHolder holder) {
    
        flag=false;

    }

    @Override
    public void run() {
    
        while (flag){
    
            //[184, 368, 552, 736, 920, 1104, 1288, 1472]

            try {
    
                doDraw();
// long drawEndTime = System.currentTimeMillis();
                //  Sleep time is the duration of each frame of animation minus the time taken to draw one frame 
                Thread.sleep(100);
            } catch (InterruptedException e) {
    
                e.printStackTrace();
            }
        }


    }

    private int s=0;
    private float angle=0.f;
    public void doDraw() {
    
        // angle 
        angle+=10;


        mCanvas = mHolder.lockCanvas(); //  Get the canvas object , Start painting on the canvas .

        Log.i("aa",bitmap.getWidth()+" "+bitmap.getHeight());
        //mCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
        //mCanvas.drawRGB(0, 0, 0); //  Fill the canvas with black 
        // eliminate canvas canvas 
        //  Method 1 ：
        mCanvas.drawColor(0,PorterDuff.Mode.CLEAR);

// Matrix matrix=new Matrix();
// // // Image translation 
//
// matrix.setTranslate(0,0);
// //matrix.preScale(0.5f,0.5f,bitmap.getWidth()/2,bitmap.getHeight()/2);
 matrix.preScale(0.5f,0.5f);
// // Image rotation angle and rotation center 
// matrix.preRotate(angle,bitmap.getWidth()/2,bitmap.getHeight()/2);
// mCanvas.drawBitmap(bitmap,matrix,null);





        try{
    

            for (MainActivity.Data p:points){
    
                int w=bitmap.getWidth();
                int h=bitmap.getHeight();

                rate=p.L*2f/w;

// rate+=(float) p.L/500f;

                Log.i("aa","pll="+p.L);

                Matrix matrix = new Matrix();
                // Image zoom 

// // Image translation 
                matrix.setTranslate(p.X-w/2,p.Y-h/2);
                matrix.preScale(rate,rate,w/2,h/2);
                // Image rotation angle and rotation center 
                matrix.preRotate(angle,w/2,h/2);

                mCanvas.drawBitmap(bitmap,matrix,null);

            }

        } catch (Exception e) {
    
            e.printStackTrace();
        }


        // Capture image area 
        //Rect srcRect=new Rect(s*(bitmap.getWidth()/8),0,(s+1)*bitmap.getWidth()/8,bitmap.getHeight());
        // Screen painting area 
// bb-=0.01;
        //Rect dstRect=new Rect(500,0,(int)(bitmap.getWidth()/8*bb)+500,(int)(bitmap.getHeight()*bb));

// s+=1;
// if (s==8){
    
// s=0;
// }

        //Log.i("aa",""+bitmap.getWidth()+" "+bitmap.getHeight());
        //mCanvas.rotate(45);// Rotate the canvas clockwise 

        //  Rotate the picture   action 
// Matrix matrix = new Matrix();
// matrix.postScale(0.5f, 0.5f);

        // Image translation 
        //matrix.setTranslate(bitmap.getWidth()/2,bitmap.getHeight()/2);
        // Image rotation 
// matrix.preRotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);

        //  Create a new picture 
// Bitmap resizedBitmap = Bitmap.createBitmap(bitmap, 0, 0,
// bitmap.getWidth(), bitmap.getHeight(), matrix, true);
//
// bb+=5;


// Rect srcRect=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());
// Rect dstcRect=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());


        //mCanvas.rotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);

// mCanvas.drawBitmap(resizedBitmap,srcRect,dstcRect,null);
// mCanvas.drawBitmap(bitmap,matrix,null);



// // Image translation 
// matrix.setTranslate(bitmap.getWidth()/2,bitmap.getHeight()/2);
// // Image rotation angle and rotation center 
// matrix.preRotate(bb,bitmap.getWidth()/2,bitmap.getHeight()/2);
// // Image zoom 
// matrix.postScale(1.5f,1.5f);
//
// mCanvas.drawBitmap(bitmap,matrix,null);

// Rect srcRect2=new Rect(0,0,bitmap.getWidth(),bitmap.getHeight());
// Rect dstcRect2=new Rect(500,500,bitmap.getWidth()+500,bitmap.getHeight()+500);




        mHolder.unlockCanvasAndPost(mCanvas); //  Finish painting , Display the canvas on the screen 
    }

// @Override
// public boolean onTouchEvent(MotionEvent event) {
    
// p.setARGB((int) (Math.random()*255),(int)(Math.random()*255),(int)(Math.random()*255),(int)(Math.random()*255));
//
// if (event.getAction()==MotionEvent.ACTION_DOWN){
    
// x = (int) event.getX(); //  Get the corresponding... When the screen is touched X Axis coordinates 
// y = (int) event.getY(); //  Get the corresponding... When the screen is touched Y Axis coordinates 
// Point neepoint=new Point(x,y);
// points.add(neepoint);
// }
//
// return false;
// }
}

MainActivity.java Code ：

package com.myapp.mycamera;

import androidx.annotation.NonNull;
import androidx.appcompat.app.AlertDialog;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.Camera;
import androidx.camera.core.CameraControl;
import androidx.camera.core.CameraSelector;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.ImageCaptureException;
import androidx.camera.core.ImageProxy;
import androidx.camera.core.Preview;
import androidx.camera.extensions.HdrImageCaptureExtender;
import androidx.camera.lifecycle.ProcessCameraProvider;
import androidx.camera.view.PreviewView;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import androidx.lifecycle.LifecycleOwner;

import android.Manifest;
import android.annotation.SuppressLint;
import android.content.DialogInterface;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.res.Configuration;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.graphics.Paint;
import android.graphics.Point;
import android.graphics.Rect;
import android.graphics.YuvImage;
import android.media.Image;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.util.Size;
import android.view.MotionEvent;
import android.view.View;
import android.widget.Button;
import android.widget.ImageView;
import android.widget.TextView;
import android.widget.Toast;

import com.google.common.util.concurrent.ListenableFuture;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;


public class MainActivity extends AppCompatActivity {
    

    // Used to load the 'native-lib' library on application startup.
// static {
    
// System.loadLibrary("native-lib");
// }

    private SSd sSdnet=new SSd();

    private Executor executor = Executors.newSingleThreadExecutor();
    private final String[] REQUIRED_PERMISSIONS = new String[]{
    "android.permission.CAMERA", "android.permission.WRITE_EXTERNAL_STORAGE"};
    private int REQUEST_CODE_PERMISSIONS = 1001;

    private Button btn;
    private Button btn2;
    private MyView2 myView2;
    private int back=1;
    private ImageView img;
    private TextView txt;

    Mypreview mPreviewView;

    private int[] pics= {
    R.drawable.a1,R.drawable.a2,R.drawable.a3};
    private int pic_id=0;
    private Configuration mConfiguration;
    @Override
    protected void onCreate(Bundle savedInstanceState) {
    
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        // Create pop ups 
        AlertDialog alertDialog=new AlertDialog.Builder(MainActivity.this).create();
        alertDialog.setCancelable(false);
        alertDialog.setTitle(" Friendship tips ");
        alertDialog.setMessage(" There are more interesting things in the application center app, Welcome to the download experience . If you encounter bug It can be sent to the mailbox [email protected]");
        alertDialog.setButton(DialogInterface.BUTTON_NEGATIVE, " got it ",
                new DialogInterface.OnClickListener() {
    
                    @Override
                    public void onClick(DialogInterface dialog, int which) {
    
                        Log.i("aa"," got it ");
                    }
                });
        alertDialog.setButton(DialogInterface.BUTTON_POSITIVE, " Application Center ",
                new DialogInterface.OnClickListener() {
    
                    @Override
                    public void onClick(DialogInterface dialog, int which) {
    
                        Intent intent=new Intent(MainActivity.this,DownLoad.class);
                        startActivity(intent);
                    }
                });
        alertDialog.show();

        boolean init=sSdnet.Init(getAssets());

        Log.i("aa",""+init);

        // Get configuration information of settings 
        mConfiguration = this.getResources().getConfiguration();



        mPreviewView = findViewById(R.id.previewView);
        img=findViewById(R.id.image);
// Bitmap bb=BitmapFactory.decodeResource(getResources(),R.drawable.pic);
//
// SSd.Obj[] outs=sSdnet.Detect(bb,false);
// Log.i("aa",outs+"");


        txt=findViewById(R.id.txt);

        if(allPermissionsGranted()){
    
            startCamera(); //start camera if permission has been granted by user
        } else{
    
            ActivityCompat.requestPermissions(this, REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS);
        }

        btn=findViewById(R.id.btn);
        btn.setOnClickListener(new View.OnClickListener() {
    
            @Override
            public void onClick(View v) {
    
                if (camea_id==0){
    
                    camea_id=1;
                }else camea_id=0;
                bindPreview(cameraProvider);
            }
        });

        myView2=findViewById(R.id.myview);
        btn2=findViewById(R.id.btn2);
        btn2.setOnClickListener(new View.OnClickListener() {
    
            @Override
            public void onClick(View v) {
    
                pic_id+=1;
                if (pic_id>2){
    
                    pic_id=0;
                }
                Bitmap bitmap2= BitmapFactory.decodeResource(getResources(),pics[pic_id]);
                myView2.bitmap=bitmap2;

            }
        });

    }

    private ProcessCameraProvider cameraProvider;
    private void startCamera() {
    

        final ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(this);

        cameraProviderFuture.addListener(new Runnable() {
    
            @Override
            public void run() {
    
                try {
    
                    cameraProvider = cameraProviderFuture.get();
                    bindPreview(cameraProvider);


                } catch (ExecutionException | InterruptedException e) {
    
                    // No errors need to be handled for this Future.
                    // This should never be reached.
                }
            }
        }, ContextCompat.getMainExecutor(this));
    }

    long t1=0;
    long t2=0;
    private int camea_id=1;
    private Bitmap bmp;
    private CameraControl cameraControl;
    private boolean su;
    private boolean heng;
    void bindPreview(@NonNull ProcessCameraProvider cameraProvider) {
    

        Preview preview = new Preview.Builder()
                .build();

        @SuppressLint("WrongConstant") CameraSelector cameraSelector = new CameraSelector.Builder()
                .requireLensFacing(camea_id)
                .build();

        ImageAnalysis imageAnalysis = new ImageAnalysis.Builder()
                .build();
        //imageAnalysis.setAnalyzer(cameraExecutor, new MyAnalyzer());
        imageAnalysis.setAnalyzer(executor, new ImageAnalysis.Analyzer() {
    
            @Override
            public void analyze(@NonNull ImageProxy image) {
    
                runOnUiThread(() ->{
    

                    // Get the current screen status 
                    int ori = mConfiguration.orientation; // Get screen orientation 
                    if (ori == mConfiguration.ORIENTATION_LANDSCAPE) {
    
                        // Horizontal screen 
                        heng=true;
                        su=false;
                    } else if (ori == mConfiguration.ORIENTATION_PORTRAIT) {
    
                        // Vertical screen 
                        su=true;
                        heng=false;
                    }


                    t1=t2;
                    t2=System.currentTimeMillis();
                    long fps=1000/(t2-t1);
                    txt.setText("FPS:"+fps);

                    int rotationDegrees = image.getImageInfo().getRotationDegrees();
// Log.i("aa","angle1="+rotationDegrees);
                    // Rotation Angle 
                    int rotation = mPreviewView.getDisplay().getRotation();
// Log.i("aa","angle2="+rotation);



                    //yuv Image data to bitmap
                    ImageProxy.PlaneProxy[] planes = image.getPlanes();

                    //cameraX  obtain yuv
                    ByteBuffer yBuffer = planes[0].getBuffer();
                    ByteBuffer uBuffer = planes[1].getBuffer();
                    ByteBuffer vBuffer = planes[2].getBuffer();

                    int ySize = yBuffer.remaining();
                    int uSize = uBuffer.remaining();
                    int vSize = vBuffer.remaining();

                    byte[] nv21 = new byte[ySize + uSize + vSize];

                    yBuffer.get(nv21, 0, ySize);
                    vBuffer.get(nv21, ySize, vSize);
                    uBuffer.get(nv21, ySize + vSize, uSize);
                    // obtain yuvImage
                    YuvImage yuvImage = new YuvImage(nv21, ImageFormat.NV21, image.getWidth(), image.getHeight(), null);
                    // Output stream 
                    ByteArrayOutputStream out = new ByteArrayOutputStream();
                    // Compression write out
                    yuvImage.compressToJpeg(new Rect(0, 0, yuvImage.getWidth(), yuvImage.getHeight()), 50, out);
                    // Turn array 
                    byte[] imageBytes = out.toByteArray();
                    // Generate bitmap
                    Bitmap bmp = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);

                    // rotate bitmap
                    Bitmap rotateBitmap=null;
                    if (camea_id==1 && su){
    
                        rotateBitmap = rotateBitmap(bmp, 90);
                    }else if(camea_id==0 && su){
    
                        rotateBitmap = rotateBitmap(bmp, 270);
                    }else if(camea_id==1 && heng){
    
                        rotateBitmap=bmp;
                    }else {
    
                        rotateBitmap=rotateBitmap(bmp, 0);
                    }


                    Bitmap bmp2=rotateBitmap.copy(Bitmap.Config.ARGB_8888, true);

                    SSd.Obj[] outcome=sSdnet.Detect(bmp2,false);
                    if(outcome!=null){
    
                        ArrayList<Data> datas=new ArrayList<>();
                        for (int i=0;i<outcome.length;i++){
    
                            Data data=new Data();

                            float x= outcome[i].x*(float)myView2.getWidth();
                            float y= outcome[i].y*(float)myView2.getHeight();
                            float w=outcome[i].w*(float)myView2.getWidth();
                            float h=outcome[i].h*(float)myView2.getHeight();
                            data.X=(int)x;
                            data.Y=(int)y;
                            data.L=(int)((w+h)/2);
                            datas.add(data);
                            myView2.points=datas;

                            Log.i("aa","ww="+x+" "+y+" "+w+" "+h);
                        }

                    }else {
    
                        ArrayList<Data> datas=new ArrayList<>();
                        myView2.points=datas;
                    }


// Canvas canvas = new Canvas( bmp2 );
// Paint paint = new Paint();
// paint.setColor( Color.RED );
// paint.setStrokeWidth( 10 );
// canvas.drawRect( 20,40,200,400,paint );
// img.setImageBitmap(bmp2);
                    // close 
                    image.close();

                });


            }
        });

        ImageCapture.Builder builder = new ImageCapture.Builder();

        //Vendor-Extensions (The CameraX extensions dependency in build.gradle)
        HdrImageCaptureExtender hdrImageCaptureExtender = HdrImageCaptureExtender.create(builder);

        // Query if extension is available (optional).
        if (hdrImageCaptureExtender.isExtensionAvailable(cameraSelector)) {
    
            // Enable the extension if available.
            hdrImageCaptureExtender.enableExtension(cameraSelector);
        }

        final ImageCapture imageCapture = builder
                .setTargetRotation(this.getWindowManager().getDefaultDisplay().getRotation())
                .build();

        preview.setSurfaceProvider(mPreviewView.createSurfaceProvider());

        try {
    
            cameraProvider.unbindAll();
            Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner)this, cameraSelector, preview, imageAnalysis, imageCapture);
            cameraControl=camera.getCameraControl();
            mPreviewView.cameraControl=cameraControl;
// cameraControl.setLinearZoom(mPreviewView.rate);

        } catch (Exception e) {
    
            e.printStackTrace();
        }


    }

    private Bitmap rotateBitmap(Bitmap origin, float alpha) {
    
        if (origin == null) {
    
            return null;
        }
        int width = origin.getWidth();
        int height = origin.getHeight();
        Matrix matrix = new Matrix();
        matrix.setRotate(alpha);
        if (camea_id==0){
    
            matrix.postScale(-1,1);
        }
        //  Rotate around where you are 
        Bitmap newBM = Bitmap.createBitmap(origin, 0, 0, width, height, matrix, false);
        if (newBM.equals(origin)) {
    
            return newBM;
        }
        origin.recycle();
        return newBM;
    }

    // Get permission function 
    private boolean allPermissionsGranted(){
    
        for(String permission : REQUIRED_PERMISSIONS){
    
            if(ContextCompat.checkSelfPermission(this, permission) != PackageManager.PERMISSION_GRANTED){
    
                return false;
            }
        }
        return true;
    }


    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
    

        if(requestCode == REQUEST_CODE_PERMISSIONS){
    
            if(allPermissionsGranted()){
    
                startCamera();
            } else{
    
                Toast.makeText(this, "Permissions not granted by the user.", Toast.LENGTH_SHORT).show();
                this.finish();
            }
        }
    }

    public class Data{
    
        int X;
        int Y;
        int L;
    }
}

3. Project download address ：https://github.com/qwhh11/MyCamera