Desktop icon recognition based on OpenCV

One 、 Necessary Libraries

numy and opencv. Use it directly pip Can be installed ;

Two 、 Screenshot in the designated area

Post code directly ：

from ctypes import windll, byref, c_ubyte
from ctypes.wintypes import RECT, HWND
import numpy as np
import time
from win32com.client import Dispatch
# import mouse as mouse
GetDC = windll.user32.GetDC
CreateCompatibleDC = windll.gdi32.CreateCompatibleDC
GetClientRect = windll.user32.GetClientRect
CreateCompatibleBitmap = windll.gdi32.CreateCompatibleBitmap
SelectObject = windll.gdi32.SelectObject
BitBlt = windll.gdi32.BitBlt
SRCCOPY = 0x00CC0020
GetBitmapBits = windll.gdi32.GetBitmapBits
DeleteObject = windll.gdi32.DeleteObject
ReleaseDC = windll.user32.ReleaseDC

#  prevent UI Zooming in causes the screenshot to be incomplete 
#windll.user32.SetProcessDPIAware()

def capture(handle: HWND):
    """ Screenshot of window client area 

    Args:
        handle (HWND):  Window handle to screenshot 

    Returns:
        numpy.ndarray:  Screenshot data 
    """
    #  Get the size of the client area of the window 
    r = RECT()
    GetClientRect(handle, byref(r))
    width, height = r.right, r.bottom
    print("width,height:", width, height)
    #  Start screenshot 
    dc = GetDC(handle)
    cdc = CreateCompatibleDC(dc)
    #  Create... In memory , Prevent the screen from blinking when taking screenshots , meanwhile , If you know the approximate range of icons that need to be recognized , Then you can 
    #  In this place will width and height Change to the width and height information of the corresponding area 
    bitmap = CreateCompatibleBitmap(dc,  width, height)
    SelectObject(cdc, bitmap)
    #  Here you can set the area of the screenshot , It should correspond to the width and height above , The subsequent width and height should correspond , Otherwise, it will not match 
    BitBlt(cdc, 0, 0, width, height, dc, 0, 0, SRCCOPY)
    #  The screenshot is BGRA array , Therefore, the total number of elements needs to be multiplied by 4
    total_bytes =  width*height*4
    buffer = bytearray(total_bytes)
    byte_array = c_ubyte*total_bytes
    GetBitmapBits(bitmap, total_bytes, byte_array.from_buffer(buffer))
    DeleteObject(bitmap)
    DeleteObject(cdc)
    ReleaseDC(handle, dc)
    #  The returned screenshot data is numpy.ndarray
    return np.frombuffer(buffer, dtype=np.uint8).reshape( width, height, 4)

if __name__ == "__main__":
    import cv2
    op=Dispatch("op.opsoft")
    handle = windll.user32.FindWindowW(None, " Here is the window handle or name ")
    #  Make sure that the client area of the game window is 1334×750
    print(handle)
    image = capture(handle)
    #  Turn to grayscale 
    gray = cv2.cvtColor(image, cv2.COLOR_BGRA2GRAY)
    #  Read the picture , And keep Alpha passageway , To eliminate background interference , You need to set the background of the source comparison image to be transparent , namely Alpha Channel set to 0.
    # Here replace it with your own figure you want to catch , It can be jpg Format screenshot 
    template = cv2.imread('23.PNG', cv2.IMREAD_UNCHANGED)
    #  Turn to grayscale 
    template_gray = cv2.cvtColor(template, cv2.COLOR_BGRA2GRAY)
    #  Take out Alpha passageway 
    alpha = template[:,:,3]

    #  Template matching , take alpha As mask,TM_CCORR_NORMED The calculation result range of the method is [0, 1], The closer the 1 More matching  TM_CCOEFF_NORMED
    result = cv2.matchTemplate(gray, template_gray, cv2.TM_CCOEFF_NORMED,mask=alpha)

    # TM_SQDIFF  Square error matching ： This method uses square difference to match ; The best match value is 0; Match the worse , The larger the match value .
    # TM_CCORR  Correlation matching ： The method adopts multiplication operation ; The larger the number, the better the match .
    # TM_CCOEFF  Correlation coefficient matching method ：1 A perfect match ;-1 Represents the worst match .
    # TM_SQDIFF_NORMED  Normalized square error matching method 
    # TM_CCORR_NORMED  Normalized correlation matching method 
    # TM_CCOEFF_NORMED  Normalized correlation coefficient matching method 
    # result = cv2.matchTemplate(gray, template, cv2.TM_CCOEFF_NORMED)

    print(cv2.minMaxLoc(result))
    #  Get the maximum and minimum values in the result and their coordinates 
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
    #  Coordinates that match the maximum , top left corner 
    top_left = max_loc
    print(" Upper left coordinate value ：",top_left[0],top_left[1])
    #  Height and width of formwork 
    h, w = template.shape[:2]
    #  The coordinates of the lower right corner 
    bottom_right = top_left[0] + w, top_left[1] + h
    if max_val>0.8:
        #  Draw a red box at the matching position in the window screenshot , stay image Draw the upper left corner as top_left, The lower right corner is bottom_right Rectangle of coordinates 
        cv2.rectangle(image, top_left, bottom_right, (0,0,255), 2)
        # Desert plug-in mouse movement , You can also use win32 Mouse control for 
        op.MoveTo(top_left[0]+4,top_left[1]+4)
    cv2.imshow('Match Template', image)
    cv2.waitKey()

then , According to this, it can be changed into automatic operation .