Test Development Notes

Memo

---

SHELL Script

#!/bin/bash

#==========================================
# Filter grayscale image and color image 
for img in Image50K/*.JPEG ; do
c=`file $img | awk '{print $NF}'`
if [ $c == "1" ];then
        echo $img
        cp $img Image50K.gray/.
else 
        cp $img Image50K.color/.
fi
done

#==========================================
# Extract a certain number of pictures to the specified path , Quantitative use of network testing 
i=0
n=2000
mkdir -p  img$n
for img in  Image50K.color/*.JPEG;do
	cp $img  img$n/.
	i=$(($i+1))
	[ $i -gt $n ] && break
done
#==========================================
# Traverse the list 
base_list=`cat <<EOF "./Classify/model/MobileNet-v1/mobilenet_merge_bn" "./Classify/model/MobileNet-v2/mobilenet-v2" "./Classify/model/Inception-v1/inception_v1_merge_bn" "./Classify/model/DenseNet121/densenet-121" "./Classify/model/ResNet50/ResNet50_train_val_merge_bn" "./Classify/model/SENet/se-resnet50_merge_bn" "./Classify/model/VGG16/VGG16_train_val" EOF `
for base_path in $base_list;do
    echo $base_path
done
#===============================================================================
# Determine whether the first one in the list is a file , Extract the file 
awk '{if(system("test -f " $1)==0) print $0}' <list_val.txt  >list_val_color.txt

#===============================================================================
# Random extraction n That's ok 
shuf list_val.txt | head -n 100

#================================================================================

base_list=`cat <<EOF ./Classify/model/MobileNet-v2/mobilenet-v2 ./Classify/model/MobileNet-v1/mobilenet_merge_bn ./Classify/model/DenseNet121/densenet-121 ./Classify/model/Inception-v1/inception_v1_merge_bn ./Classify/model/SqueezeNet_V1.1/squeezenet_train_val ./Classify/model/VGG16/VGG16_train_val ./Classify/model/SENet/se-resnet50_merge_bn ./Classify/model/ResNet50/ResNet50_train_val_merge_bn EOF `

for base_path in $base_list;do
    proto=${base_path}.prototxt
    model=${base_path}.caffemodel
    logfile=`basename ${
     base_path}.log`

    echo $logfile

    ./caffe_predict.py  ${proto}  ${model} ${logfile}  &> predict.log

    tr -d "\[\]," <$logfile >${logfile}.1

    mv ${logfile}.1  ${logfile}

    ./readlog.py ${logfile}
done

Python Script

#=====================================================================
#  Analyzing log files , Calculation top10
#!/usr/bin/python3
# -*- coding: UTF-8 -*-

import re
import sys,os
import numpy as np

counts=np.zeros([11],dtype=float)

def search(topk):
    label=topk[-1]
    pos=0
    while True:
        if topk[pos]==label:
            return pos
        pos=pos+1

logfile=sys.argv[1]

with open(logfile,'r') as f:
    for line in f.readlines():
        line.strip()
        topk=[int(i) for i in line.split()]
        pos=search(topk)
        counts[pos]=counts[pos]+1

print(logfile,": counts",counts)
print(logfile," topk acc: ", np.cumsum(counts)/np.sum(counts))

#=======================================================================
# Judge image yes 1 passageway , It's slow 
#!/usr/bin/env python3
from PIL import Image
import sys,os

img=Image.open(sys.argv[1])
if len(img.split())==1:
        print(sys.argv[1])
#========================================================================
#caffe_predict.py,  test caffe Model 
#!/usr/bin/env python3
import sys
import numpy as np
import cv2
import caffe
import time
from preprocess import *

# select GPU to predict
caffe.set_mode_gpu()
caffe.set_device(0)


# Preprocessing parameter dictionary corresponding to the model 
transform_params={
     \
'se-resnet50_merge_bn.log':(225,0.0170000009239,103.940002441,116.779998779,123.680000305), \
'mobilenet-v2.log':(224,0.017,103.94,116.78,123.68), \
'mobilenet_merge_bn.log':(224,0.0170000009239,103.940002441,116.779998779,123.680000305), \
'densenet-121.log':(224,0.017,103.94,116.78,123.68),  \
'inception_v1_merge_bn.log':(224,1.0,128.0,128.0,128.0), \
'squeezenet_train_val.log':(227,1.0,103.939,116.779,123.68), \
'VGG16_train_val.log':(224,1.0,103.939,116.779,123.68), \
'ResNet50_train_val_merge_bn.log':(224,1.0,103.939,116.779,123.68), \
}

test_data_path = './caffe.dir//ILSVRC2012_val/'

model_file=sys.argv[1]
pretrained=sys.argv[2]
log_file=sys.argv[3]

transform=transform_params[log_file]
print(transform)

net = caffe.Net(model_file, pretrained, caffe.TEST)

crop_size=transform[0]
scale=transform[1]
mean=transform[2:]

f = open(test_data_path + "list_val.txt")
right = 0
i = 0

with open(log_file,'w') as fh:
    #for line in f.readlines()[:500]:
    for line in f.readlines():
        line = line.strip().split(" ")
        filename = line[0]
        
        label =line[1]
        image_path = test_data_path + filename

        img = cv2.imread(image_path)
        imgresize = resize(img, 256)
      
        imgcrop=center_crop(imgresize,crop_size)
        imgcrop = imgcrop.astype(np.float64)
        stdvalue = 1 / scale                         # The std value is in the prototxt
        std = (stdvalue, stdvalue, stdvalue)
        imgcrop -= mean
        imgcrop/=std

        imgcrop = imgcrop.astype(np.float64)

        #HWC->CHW
        net.blobs['data'].data[...] = imgcrop.transpose((2, 0, 1))


        start = time.time()
        out = net.forward() # output
        end = time.time()
        running_time = end - start
        print("filename:",filename)

        output_prob = out['prob'][0]
        print('predicted class is:', output_prob.argmax())

        top_k = net.blobs['prob'].data[0].flatten().argsort()[-1:-11:-1]
        ll=top_k.tolist()
        ll.append(int(label))
        print(ll)
        fh.write(str(ll)+'\n')
        print('time cost : %.5f s' % running_time)

Python call so Dynamic library

#include<cstdio>
#include<string>
#include<iostream>

/*********compile&link***********************/
/* g++ -c -Wall -Werror -fpic t.cpp g++ -shared -o libtest.so t.o */


/******Python************/
/* from ctypes import * soo=cdll.LoadLibrary("./libtest.so") func=soo._Z4funcPKcS0_i func(b'hello',b'world',4) */

//using namespace std;
int
func(const char *str1, const char *str2, int n)
{
    
    printf("string_1: %s\n",str1);
    //cout<<"string_1:"<<str1<<endl;
    printf("string_2: %s\n",str2);
    printf("n=%d\n",n);

    return 1;
}

System configuration related

# add to NFS Shared disk , edit /etc/fstab
# Add a row : x.x.x.x:/data /data nfs defaults 0 0
# Installation may be required mount.nfs
apt search mount.nfs
sudo apt install mount.nfs
# load nfs Shared disk 
sudo mount -a

# download deb package 
apt-get download xxx.deb
# take deb Expand the package to the specified path 
dpkg -X xxx.deb ./path

# Add dynamic library directory 
# edit /etc/ld.so.conf.d/slurm.conf
# Refresh ld caches
ldconfig
# Add node name 
echo "192.168.11.75 worker02" >>/etc/hosts
# Display node hardware resource information 
slurmd -C
#drain State processing , Operation on the control node 
sudo scontrol update NodeName=worker02 State=DOWN Reason=hung_completing
sudo scontrol update NodeName=worker02 State=RESUME

# function slurmd, Debugging information 
sudo `which slurmd` -D -vvvv

# debugging ssh
ssh -vvv [email protected]
{
    ssh -vvv [email protected];} 2>1.log