Bert whitening vector dimension reduction and its application

Reference resources ：https://kexue.fm/archives/8069
https://kexue.fm/archives/9079
https://zhuanlan.zhihu.com/p/531476789

Input ：vv Is a three-dimensional matrix composed of multiple vectors

result ：v_data1 256 dimension

def compute_kernel_bias(vecs, n_components=256):
    """ Calculation kernel and bias
    vecs.shape = [num_samples, embedding_size],
     The last transformation ：y = (x + bias).dot(kernel)
    """
    mu = vecs.mean(axis=0, keepdims=True)
    cov = np.cov(vecs.T)
    # print(cov)
    u, s, vh = np.linalg.svd(cov)
    print(np.diag(1 / np.sqrt(s) ))
    W = np.dot(u, np.diag(1 / np.sqrt(s)))
    return W[:, :n_components], -mu


def transform_and_normalize(vecs, kernel=None, bias=None):
    """  The final vector is normalized 
    """
    if not (kernel is None or bias is None):
        vecs = (vecs + bias).dot(kernel)
    return vecs / (vecs**2).sum(axis=1, keepdims=True)**0.5


v_data = np.array(vv[0])    ## vv[0] A two-dimensional matrix composed of multiple vectors , If you input a vector, the two-dimensional matrix calculation will report an error 

kernel,bias=compute_kernel_bias(v_data)
# print(kernel,bias)
v_data1=transform_and_normalize(v_data, kernel=kernel, bias=bias)

*** A single vector on the line will be calculated as a whole kernel,bias use , direct transform_and_normalize(v_data, kernel=kernel, bias=bias) Just go

import numpy as np
data = np.random.rand(5,768)
print('data.shape = ')
print(data.shape,data)
def compute_kernel_bias(vecs):
    """ Calculation kernel and bias
    vecs.shape = [num_samples, embedding_size],
     The last transformation ：y = (x + bias).dot(kernel)
    """
    mu = vecs.mean(axis=0, keepdims=True)
    cov = np.cov(vecs.T)
    u, s, vh = np.linalg.svd(cov)
    W = np.dot(u, np.diag(1 / np.sqrt(s)))
    return W, -mu

def transform_and_normalize(vecs, kernel=None, bias=None):
    """ Apply transformation , And then standardize 
    """
    if not (kernel is None or bias is None):
        vecs = (vecs + bias).dot(kernel)
    return vecs / (vecs**2).sum(axis=1, keepdims=True)**0.5

kernel,bias = compute_kernel_bias(data)
kernel = kernel[:,:64]
print('kernel.shape = ')
print(kernel.shape)
print('bias.shape = ')
print(bias.shape)
data = transform_and_normalize(data, kernel, bias)
print('data.shape = ')
print(data.shape,data)

Dimension reduction of single vector on line

data1 = np.random.rand(1,768)
data1_1 = transform_and_normalize(data1, kernel, bias)