Data cleaning - ingest using es

 通常es产品里面,数据清洗的logstash,Here use anotheringest做简单的数据处理

Here is the data separated by comma,变成数组

PUT spring_blogs/_doc/1
{
  "title":"Introducing spring framework......",
  "tags":"spring,spring boot,spring cloud",
  "content":"You konw, for spring framework"
}

POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "to split blog tags",
    "processors": [
      {
        "split": {
          "field": "tags",
          "separator": ","
        }
      }
    ]
  },
  "docs": [
    {
      "_index": "index",
      "_id": "id",
      "_source": {
        "title": "Introducing spring framework......",
        "tags": "spring,spring boot,spring cloud",
        "content": "You konw, for spring framework"
      }
    },
    {
      "_index": "index",
      "_id": "idxx",
      "_source": {
        "title": "Introducing cloud computering",
        "tags": "docker,k8s,ingrest",
        "content": "You konw, for cloud"
      }
    }
  ]
}

# 为ES添加一个 Pipeline
PUT _ingest/pipeline/spring_blog_pipeline
{
  "description": "a spring blog pipeline",
  "processors": [
      {
        "split": {
          "field": "tags",
          "separator": ","
        }
      },

      {
        "set":{
          "field": "views",
          "value": 0
        }
      }
    ]
}

#查看Pipleline
GET _ingest/pipeline/spring_blog_pipeline

#测试pipeline
POST _ingest/pipeline/spring_blog_pipeline/_simulate
{
  "docs": [
    {
      "_source": {
        "title": "Introducing cloud computering",
        "tags": "docker,k8s,ingrest",
        "content": "You konw, for cloud"
      }
    }
  ]
}

DELETE spring_blogs

PUT spring_blogs/_doc/1
{
  "title":"Introducing spring framework......",
  "tags":"spring,spring boot,spring cloud",
  "content":"You konw, for spring framework"
}

#使用pipeline更新数据
PUT spring_blogs/_doc/2?pipeline=spring_blog_pipeline
{
  "title": "Introducing cloud computering",
  "tags": "docker,k8s,ingrest",
  "content": "You konw, for cloud"
}

POST spring_blogs/_search


#增加update_by_query的条件
POST spring_blogs/_update_by_query?pipeline=spring_blog_pipeline
{
    "query": {
        "bool": {
            "must_not": {
                "exists": {
                    "field": "views"
                }
            }
        }
    }
}

The final processed data

{
  "took" : 0,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 2,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "spring_blogs",
        "_id" : "2",
        "_score" : 1.0,
        "_source" : {
          "title" : "Introducing cloud computering",
          "content" : "You konw, for cloud",
          "views" : 0,
          "tags" : [
            "docker",
            "k8s",
            "ingrest"
          ]
        }
      },
      {
        "_index" : "spring_blogs",
        "_id" : "1",
        "_score" : 1.0,
        "_source" : {
          "title" : "Introducing spring framework......",
          "content" : "You konw, for spring framework",
          "views" : 0,
          "tags" : [
            "spring",
            "spring boot",
            "spring cloud"
          ]
        }
      }
    ]
  }
}

还可以使用Script Prcessor,This degree of freedom is greater,Can handle slightly more complex data

POST _ingest/pipeline/_simulate
{
  "pipeline": {
    "description": "to split spring blog tags",
    "processors": [
      {
        "split": {
          "field": "tags",
          "separator": ","
        }
      },
      {
        "script": {
          "source": """
          if(ctx.containsKey("title")){
            ctx.content_length = ctx.title.length();
          }else{
            ctx.content_length=0;
          }


          """
        }
      },
      {
        "set": {
          "field": "views",
          "value": 0
        }
      }
    ]
  },
  "docs": [
    {
      "_index": "index",
      "_id": "id",
      "_source": {
        "title": "Introducing spring framework......",
        "tags": "spring,spring boot,spring cloud",
        "content": "You konw, for spring framework"
      }
    },
    {
      "_index": "index",
      "_id": "idxx",
      "_source": {
        "title": "Introducing cloud computering",
        "tags": "docker,k8s,ingrest",
        "content": "You konw, for cloud"
      }
    }
  ]
}