moloch/db/db.pl at master · newdominic/moloch

Repository navigation

#!/usr/bin/perl
# This script can initialize, upgrade or provide simple maintenance for the
# moloch elastic search db
# Schema Versions
#  0 - Before this script existed
#  1 - First version of script; turned on strict schema; added lpms, fpms; added
#      many missing items that were dynamically created by mistake
#  2 - Added email items
#  3 - Added email md5
#  4 - Added email host and ip; added help, usersimport, usersexport, wipe commands
#  5 - No schema change, new rotate command, encoding of file pos is different.
#      Negative is file num, positive is file pos
#  6 - Multi fields for spi view, added xffcnt, 0.90 fixes, need to type INIT/UPGRADE
#      instead of YES
#  7 - files_v3
#  8 - fileSize, memory to stats/dstats and -v flag
#  9 - http body hash, rawus
# 10 - dynamic fields for http and email headers
# 11 - Require 0.90.1, switch from soft to node, new fpd field, removed fpms field
# 12 - Added hsver, hdver fields, diskQueue, user settings, scrub* fields, user removeEnabled
# 13 - Rename rotate to expire, added smb, socks, rir fields
# 14 - New http fields, user.views
# 15 - New first byte fields, socks user
# 16 - New dynamic plugin section
# 17 - email hasheader, db,pa,by src and dst
# 18 - fields db
# 19 - users_v3
# 20 - queries
# 21 - doc_values, new tls fields, starttime/stoptime/view
# 22 - cpu to stats/dstats
# 23 - packet lengths
# 24 - field category
# 25 - cert hash
# 26 - dynamic stats, ES 2.0
# 27 - table states
use HTTP::Request::Common;
use LWP::UserAgent;
use Data::Dumper;
use strict;
my $VERSION = 27;
my $verbose = 0;
my $PREFIX = "";
my $SHARDS = -1;
my $REPLICAS = -1;
################################################################################
sub MIN ($$) { $_[$_[0] > $_[1]] }
sub MAX ($$) { $_[$_[0] < $_[1]] }
sub commify {
    scalar reverse join ',',
    unpack '(A3)*',
    scalar reverse shift
################################################################################
sub showHelp($)
    my ($str) = @_;
    print "\n", $str,"\n\n";
    print "$0 [Global Options] <ESHOST:ESPORT> <command> [<command arguments>]\n";
    print "\n";
    print "Global Options:\n";
    print "  -v                           - Verbose, multiple increases level\n";
    print "  --prefix <prefix>            - Prefix for table names\n";
    print "\n";
    print "Commands:\n";
    print "  init [<opts>]                - Clear ALL elasticsearch moloch data and create schema\n";
    print "    --shards <shards>          - Number of shards for sessions, default number of nodes\n";
    print "    --replicas <num>           - Number of replicas for sessions, default 0\n";
    print "  upgrade [<opts>]             - Upgrade Moloch's schema in elasticsearch from previous versions\n";
    print "    --shards <shards>          - Number of shards for sessions, default number of nodes\n";
    print "    --replicas <num>           - Number of replicas for sessions, default 0\n";
    print "  wipe                         - Same as init, but leaves user database untouched\n";
    print "  info                         - Information about the database\n";
    print "  users-export <fn>            - Save the users info to <fn>\n";
    print "  users-import <fn>            - Load the users info from <fn>\n";
    print "  optimize                     - Optimize all indices in ES\n";
    print "  mv <old fn> <new fn>         - Move a pcap file in the database (doesn't change disk)\n";
    print "  rm <fn>                      - Remove a pcap file in the database (doesn't change disk)\n";
    print "  rm-missing <node>            - Remove from db any MISSING file on THIS machine for named node\n";
    print "  rm-node <node>               - Remove from db all data for node (doesn't change disk)\n";
    print "  add-missing <node> <dir>     - Add to db any MISSING file on THIS machine for named node and directory\n";
    print "  expire <type> <num> [<opts>] - Perform daily maintenance and optimize all indices in ES\n";
    print "       type                    - Same as rotateIndex in ini file = hourly,daily,weekly,monthly\n";
    print "       num                     - number of indexes to keep\n";
    print "    --replicas <num>           - Number of replicas for older sessions indices, default 0\n";
    print "  field disable <exp >         - disable a field from being indexed\n";
    print "  field enable <exp >          - enable a field from being indexed\n";
    exit 1;
################################################################################
sub waitFor
    my ($str, $help) = @_;
    print "Type \"$str\" to continue - $help?\n";
    while (1) {
        my $answer = <STDIN>;
        chomp $answer;
        last if ($answer eq $str);
        print "You didn't type \"$str\", for some reason you typed \"$answer\"\n";
################################################################################
    my ($url, $dontcheck) = @_;
    print "GET ${main::elasticsearch}$url\n" if ($verbose > 2);
    my $response = $main::userAgent->get("${main::elasticsearch}$url");
    if (($response->code == 500 && $ARGV[1] ne "init") || ($response->code != 200 && !$dontcheck)) {
      die "Couldn't GET ${main::elasticsearch}$url  the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
    my $json = from_json($response->content);
    return $json
################################################################################
    my ($url, $content, $dontcheck) = @_;
    print "POST ${main::elasticsearch}$url\n" if ($verbose > 2);
    my $response = $main::userAgent->post("${main::elasticsearch}$url", Content => $content);
    if ($response->code == 500 || ($response->code != 200 && $response->code != 201 && !$dontcheck)) {
      die "Couldn't POST ${main::elasticsearch}$url  the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
    my $json = from_json($response->content);
    return $json
################################################################################
    my ($url, $content, $dontcheck) = @_;
    print "PUT ${main::elasticsearch}$url\n" if ($verbose > 2);
    my $response = $main::userAgent->request(HTTP::Request::Common::PUT("${main::elasticsearch}$url", Content => $content));
    if ($response->code == 500 || ($response->code != 200 && !$dontcheck)) {
      print Dumper($response);
      die "Couldn't PUT ${main::elasticsearch}$url  the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?\n" . $response->content;
    my $json = from_json($response->content);
    return $json
################################################################################
sub esDelete
    my ($url, $dontcheck) = @_;
    print "DELETE ${main::elasticsearch}$url\n" if ($verbose > 2);
    my $response = $main::userAgent->request(HTTP::Request::Common::_simple_req("DELETE", "${main::elasticsearch}$url"));
    if ($response->code == 500 || ($response->code != 200 && !$dontcheck)) {
      die "Couldn't DELETE ${main::elasticsearch}$url  the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
    my $json = from_json($response->content);
    return $json
################################################################################
    my ($srci, $dsti, $type) = @_;
    my $status = esGet("/_status", 1);
    print "Copying " . $status->{indices}->{$PREFIX . $srci}->{docs}->{num_docs} . " elements from ${PREFIX}$srci/$type to ${PREFIX}$dsti/$type\n";
    my $id = "";
    while (1) {
        if ($verbose > 0) {
            local $| = 1;
            print ".";
        my $url;
        if ($id eq "") {
            $url = "/${PREFIX}$srci/$type/_search?scroll=10m&scroll_id=$id&size=500";
        } else {
            $url = "/_search/scroll?scroll=10m&scroll_id=$id";
        my $incoming = esGet($url);
        my $out = "";
        last if (@{$incoming->{hits}->{hits}} == 0);
        foreach my $hit (@{$incoming->{hits}->{hits}}) {
            $out .= "{\"index\": {\"_index\": \"${PREFIX}$dsti\", \"_type\": \"$type\", \"_id\": \"" . $hit->{_id} . "\"}}\n";
            $out .= to_json($hit->{_source}) . "\n";
        $id = $incoming->{_scroll_id};
        esPost("/_bulk", $out);
    print "\n"
################################################################################
sub esAlias
    my ($cmd, $index, $alias) = @_;
    print "Alias cmd $cmd from $index to alias $alias\n" if ($verbose > 0);
    esPost("/_aliases", '{ "actions": [ { "' . $cmd . '": { "index": "' . $PREFIX . $index . '", "alias" : "'. $PREFIX . $alias .'" } } ] }', 1);
################################################################################
sub tagsCreate
    my $settings = '
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-all"
    print "Creating tags_v2 index\n" if ($verbose > 0);
    esPut("/${PREFIX}tags_v2/", $settings);
    esAlias("add", "tags_v2", "tags");
    tagsUpdate();
################################################################################
sub tagsUpdate
    my $mapping = '
    "dynamic": "strict",
    "properties": {
      "n": {
        "type": "integer"
    print "Setting tags_v2 mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}tags_v2/tag/_mapping", $mapping);
################################################################################
sub sequenceCreate
    my $settings = '
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-all"
    print "Creating sequence index\n" if ($verbose > 0);
    esPut("/${PREFIX}sequence", $settings);
    sequenceUpdate();
################################################################################
sub sequenceUpdate
    my $mapping = '
  "sequence": {
    "_source" : { "enabled": 0 },
    "_all"    : { "enabled": 0 },
    "enabled" : 0
    print "Setting sequence mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}sequence/sequence/_mapping", $mapping);
################################################################################
sub filesCreate
    my $settings = '
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-2"
    print "Creating files_v3 index\n" if ($verbose > 0);
    esPut("/${PREFIX}files_v3", $settings);
    esAlias("add", "files_v3", "files");
    filesUpdate();
################################################################################
sub filesUpdate
    my $mapping = '
  "file": {
    "_all": {"enabled": 0},
    "_source": {"enabled": 1},
    "dynamic": "strict",
    "properties": {
      "num": {
        "type": "long",
        "index": "not_analyzed"
      "node": {
        "type": "string",
        "index": "not_analyzed"
      "first": {
        "type": "long",
        "index": "not_analyzed"
      "name": {
        "type": "string",
        "index": "not_analyzed"
      "filesize": {
        "type": "long"
      "locked": {
        "type": "short",
        "index": "not_analyzed"
      "last": {
        "type": "long",
        "index": "not_analyzed"
    print "Setting files_v3 mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}files_v3/file/_mapping", $mapping);
################################################################################
sub statsCreate
    my $settings = '
  "index": {
    "store": {
      "type": "memory"
  "settings": {
      "number_of_shards": 1,
      "number_of_replicas": 0,
      "auto_expand_replicas": "0-all"
    print "Creating stats index\n" if ($verbose > 0);
    esPut("/${PREFIX}stats", $settings);
    statsUpdate();
################################################################################
sub statsUpdate
my $mapping = '
  "stat": {
    "_all": {enabled : false},
    "_source": {enabled : true},
    "dynamic": "true",
    "properties": {
      "hostname": {
        "type": "string",
        "index": "not_analyzed"
      "currentTime": {
        "type": "long"
      "freeSpaceM": {
        "type": "long",
        "index": "no"
      "totalK": {
        "type": "long",
        "index": "no"
      "totalPackets": {
        "type": "long",
        "index": "no"
      "monitoring": {
        "type": "long",
        "index": "no"
      "totalSessions": {
        "type": "long",
        "index": "no"
      "totalDropped": {
        "type": "long",
        "index": "no"
      "deltaMS": {
        "type": "long",
        "index": "no"
      "deltaBytes": {
        "type": "long",
        "index": "no"
      "deltaPackets": {
        "type": "long",
        "index": "no"
      "deltaSessions": {
        "type": "long",
        "index": "no"
      "deltaDropped": {
        "type": "long",
        "index": "no"
      "memory": {
        "type": "long",
        "index": "no"
      "cpu": {
        "type": "integer",
        "index": "no"
      "diskQueue": {
        "type": "long",
        "index": "no"
    print "Setting stats mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}stats/stat/_mapping?pretty&ignore_conflicts=true", $mapping, 1);
################################################################################
sub dstatsCreate
    my $settings = '
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-2"
    print "Creating dstats_v1 index\n" if ($verbose > 0);
    esPut("/${PREFIX}dstats_v1", $settings);
    esAlias("add", "dstats_v1", "dstats");
    dstatsUpdate();
################################################################################
sub dstatsUpdate
my $mapping = '
  "dstat": {
    "_all": {"enabled": false},
    "_source": {"enabled": true},
    "dynamic": "true",
    "properties": {
      "nodeName": {
        "type": "string",
        "index": "not_analyzed"
      "interval": {
        "type": "short"
      "currentTime": {
        "type": "long"
      "freeSpaceM": {
        "type": "long",
        "index": "no"
      "deltaMS": {
        "type": "long",
        "index": "no"
      "deltaBytes": {
        "type": "long",
        "index": "no"
      "deltaPackets": {
        "type": "long",
        "index": "no"
      "deltaSessions": {
        "type": "long",
        "index": "no"
      "deltaDropped": {
        "type": "long",
        "index": "no"
      "monitoring": {
        "type": "long",
        "index": "no"
      "memory": {
        "type": "long",
        "index": "no"
      "cpu": {
        "type": "integer",
        "index": "no"
      "diskQueue": {
        "type": "long",
        "index": "no"
    print "Setting dstats_v1 mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}dstats_v1/dstat/_mapping?pretty&ignore_conflicts=true", $mapping, 1);
################################################################################
sub fieldsCreate
    my $settings = '
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-2"
    print "Creating fields index\n" if ($verbose > 0);
    esPut("/${PREFIX}fields", $settings);
    fieldsUpdate();
################################################################################
sub fieldsUpdate
    my $mapping = '
  "field": {
    "_all": {"enabled": 0},
    "_source": {"enabled": 1},
    "dynamic_templates": [
        "template_1": {
          "match_mapping_type": "string",
          "mapping": {
            "index": "not_analyzed"
    print "Setting fields mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}fields/field/_mapping", $mapping);
    esPost("/${PREFIX}fields/field/ip", '{
      "friendlyName": "All IP fields",
      "group": "general",
      "help": "Search all ip fields",
      "type": "ip",
      "dbField": "ipall",
      "portField": "portall",
      "noFacet": true
    esPost("/${PREFIX}fields/field/port", '{
      "friendlyName": "All port fields",
      "group": "general",
      "help": "Search all port fields",
      "type": "integer",
      "dbField": "portall",
      "regex": "(^port\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.port$)"
    esPost("/${PREFIX}fields/field/rir", '{
      "friendlyName": "All rir fields",
      "group": "general",
      "help": "Search all rir fields",
      "type": "uptermfield",
      "dbField": "all",
      "regex": "(^rir\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.rir$)"
    esPost("/${PREFIX}fields/field/country", '{
      "friendlyName": "All country fields",
      "group": "general",
      "help": "Search all country fields",
      "type": "uptermfield",
      "dbField": "all",
      "regex": "(^country\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.country$)"
    esPost("/${PREFIX}fields/field/asn", '{
      "friendlyName": "All ASN fields",
      "group": "general",
      "help": "Search all ASN fields",
      "type": "textfield",
      "dbField": "all",
      "regex": "(^asn\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.asn$)"
    esPost("/${PREFIX}fields/field/host", '{
      "friendlyName": "All Host fields",
      "group": "general",
      "help": "Search all Host fields",
      "type": "lotextfield",
      "dbField": "all",
      "regex": "(^host\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.host$)"
    esPost("/${PREFIX}fields/field/ip.src", '{
      "friendlyName": "Src IP",
      "group": "general",
      "help": "Source IP",
      "type": "ip",
      "dbField": "a1",
      "portField": "p1",
      "category": "ip"
    esPost("/${PREFIX}fields/field/port.src", '{
      "friendlyName": "Src Port",
      "group": "general",
      "help": "Source Port",
      "type": "integer",
      "dbField": "p1",
      "category": "port"
    esPost("/${PREFIX}fields/field/asn.src", '{
      "friendlyName": "Src ASN",
      "group": "general",
      "help": "GeoIP ASN string calculated from the source IP",
      "type": "textfield",
      "dbField": "as1",
      "rawField": "rawas1",
      "category": "asn"
    esPost("/${PREFIX}fields/field/country.src", '{
      "friendlyName": "Src Country",
      "group": "general",
      "help": "Source Country",
      "type": "uptermfield",
      "dbField": "g1",
      "category": "country"
    esPost("/${PREFIX}fields/field/rir.src", '{
      "friendlyName": "Src RIR",
      "group": "general",
      "help": "Source RIR",
      "type": "uptermfield",
      "dbField": "rir1",
      "category": "rir"
    esPost("/${PREFIX}fields/field/ip.dst", '{
      "friendlyName": "Dst IP",
      "group": "general",
      "help": "Destination IP",
      "type": "ip",
      "dbField": "a2",
      "portField": "p2",
      "category": "ip"
    esPost("/${PREFIX}fields/field/port.dst", '{
      "friendlyName": "Dst Port",
      "group": "general",
      "help": "Source Port",
      "type": "integer",
      "dbField": "p2",
      "category": "port"
    esPost("/${PREFIX}fields/field/asn.dst", '{
      "friendlyName": "Dst ASN",
      "group": "general",
      "help": "GeoIP ASN string calculated from the destination IP",
      "type": "textfield",
      "dbField": "as2",
      "rawField": "rawas2",
      "category": "asn"
    esPost("/${PREFIX}fields/field/country.dst", '{
      "friendlyName": "Dst Country",
      "group": "general",
      "help": "Destination Country",
      "type": "uptermfield",
      "dbField": "g2",
      "category": "country"
    esPost("/${PREFIX}fields/field/rir.dst", '{
      "friendlyName": "Dst RIR",
      "group": "general",
      "help": "Destination RIR",
      "type": "uptermfield",
      "dbField": "rir2",
      "category": "rir"
    esPost("/${PREFIX}fields/field/bytes", '{
      "friendlyName": "Bytes",
      "group": "general",
      "help": "Total number of raw bytes sent AND received in a session",
      "type": "integer",
      "dbField": "by"
    esPost("/${PREFIX}fields/field/bytes.src", '{
      "friendlyName": "Src Bytes",
      "group": "general",
      "help": "Total number of raw bytes sent by source in a session",
      "type": "integer",
      "dbField": "by1"
    esPost("/${PREFIX}fields/field/bytes.dst", '{
      "friendlyName": "Dst Bytes",
      "group": "general",
      "help": "Total number of raw bytes sent by destination in a session",
      "type": "integer",
      "dbField": "by2"
    esPost("/${PREFIX}fields/field/databytes", '{
      "friendlyName": "Data bytes",
      "group": "general",
      "help": "Total number of data bytes sent AND received in a session",
      "type": "integer",
      "dbField": "db"
    esPost("/${PREFIX}fields/field/databytes.src", '{
      "friendlyName": "Src data bytes",
      "group": "general",
      "help": "Total number of data bytes sent by source in a session",
      "type": "integer",
      "dbField": "db1"
    esPost("/${PREFIX}fields/field/databytes.dst", '{
      "friendlyName": "Dst data bytes",
      "group": "general",
      "help": "Total number of data bytes sent by destination in a session",
      "type": "integer",
      "dbField": "db2"
    esPost("/${PREFIX}fields/field/packets", '{
      "friendlyName": "Packets",
      "group": "general",
      "help": "Total number of packets sent AND received in a session",
      "type": "integer",
      "dbField": "pa"
    esPost("/${PREFIX}fields/field/packets.src", '{
      "friendlyName": "Src Packets",
      "group": "general",
      "help": "Total number of packets sent by source in a session",
      "type": "integer",
      "dbField": "pa1"
    esPost("/${PREFIX}fields/field/packets.dst", '{
      "friendlyName": "Dst Packets",
      "group": "general",
      "help": "Total number of packets sent by destination in a session",
      "type": "integer",
      "dbField": "pa2"
    esPost("/${PREFIX}fields/field/ip.protocol", '{
      "friendlyName": "IP Protocol",
      "group": "general",
      "help": "IP protocol number or friendly name",
      "type": "lotermfield",
      "dbField": "pr",
      "transform": "ipProtocolLookup"
    esPost("/${PREFIX}fields/field/id", '{
      "friendlyName": "Moloch ID",
      "group": "general",
      "help": "Moloch ID for the session",
      "type": "termfield",
      "dbField": "_id",
      "noFacet": true
    esPost("/${PREFIX}fields/field/rootId", '{
      "friendlyName": "Moloch Root ID",
      "group": "general",
      "help": "Moloch ID of the first session in a multi session stream",
      "type": "termfield",
      "dbField": "ro"
    esPost("/${PREFIX}fields/field/node", '{
      "friendlyName": "Moloch Node",
      "group": "general",
      "help": "Moloch node name the session was recorded on",
      "type": "termfield",
      "dbField": "no"
    esPost("/${PREFIX}fields/field/file", '{
      "friendlyName": "Filename",
      "group": "general",
      "help": "Moloch offline pcap filename",
      "type": "fileand",
      "dbField": "fileand"
    esPost("/${PREFIX}fields/field/payload8.src.hex", '{
      "friendlyName": "Payload Src Hex",
      "group": "general",
      "help": "First 8 bytes of source payload in hex",
      "type": "lotermfield",
      "dbField": "fb1",
      "aliases": ["payload.src"]
    esPost("/${PREFIX}fields/field/payload8.src.utf8", '{
      "friendlyName": "Payload Src UTF8",
      "group": "general",
      "help": "First 8 bytes of source payload in utf8",
      "type": "termfield",
      "dbField": "fb1",
      "transform": "utf8ToHex",
      "noFacet": true
    esPost("/${PREFIX}fields/field/payload8.dst.hex", '{
      "friendlyName": "Payload Dst Hex",
      "group": "general",
      "help": "First 8 bytes of destination payload in hex",
      "type": "lotermfield",
      "dbField": "fb2",
      "aliases": ["payload.dst"]
    esPost("/${PREFIX}fields/field/payload8.dst.utf8", '{
      "friendlyName": "Payload Dst UTF8",
      "group": "general",
      "help": "First 8 bytes of destination payload in utf8",
      "type": "termfield",
      "dbField": "fb2",
      "transform": "utf8ToHex",
      "noFacet": true
    esPost("/${PREFIX}fields/field/payload8.hex", '{
      "friendlyName": "Payload Hex",
      "group": "general",
      "help": "First 8 bytes of payload in hex",
      "type": "lotermfield",
      "dbField": "fballhex",
      "regex": "^payload8.(src|dst).hex$"
    esPost("/${PREFIX}fields/field/payload8.utf8", '{
      "friendlyName": "Payload UTF8",
      "group": "general",
      "help": "First 8 bytes of payload in hex",
      "type": "lotermfield",
      "dbField": "fballutf8",
      "regex": "^payload8.(src|dst).utf8$"
    esPost("/${PREFIX}fields/field/scrubbed.by", '{
      "friendlyName": "Scrubbed By",
      "group": "general",
      "help": "SPI data was scrubbed by",
      "type": "lotermfield",
      "dbField": "scrubby"
    esPost("/${PREFIX}fields/field/view", '{
      "friendlyName": "View Name",
      "group": "general",
      "help": "Moloch view name",
      "type": "viewand",
      "dbField": "viewand",
      "noFacet": true
    esPost("/${PREFIX}fields/field/starttime", '{
      "friendlyName": "Start Time",
      "group": "general",
      "help": "Session Start Time",
      "type": "seconds",
      "dbField": "fp"
    esPost("/${PREFIX}fields/field/stoptime", '{
      "friendlyName": "Stop Time",
      "group": "general",
      "help": "Session Stop Time",
      "type": "seconds",
      "dbField": "lp"
    esPost("/${PREFIX}fields/field/dns.status/_update", '{"doc": {"type": "uptermfield"}}', 1);
    esPost("/${PREFIX}fields/field/http.hasheader/_update", '{"doc": {regex: "^http.hasheader\\\\.(?:(?!\\\\.cnt$).)*$"}}', 1);
    esPost("/${PREFIX}fields/field/email.subject/_update", '{"doc": {"type": "textfield"}}', 1);
################################################################################
sub queriesCreate
    my $settings = '
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0,
    "auto_expand_replicas": "0-2"
    print "Creating queries index\n" if ($verbose > 0);
    esPut("/${PREFIX}queries", $settings);
    queriesUpdate();
################################################################################
sub queriesUpdate
    my $mapping = '
  "query": {
    "_all": {"enabled": 0},
    "_source": {"enabled": 1},
    "dynamic": "strict",
    "properties": {
      "name": {
        "type": "string",
        "index": "not_analyzed"
      "enabled": {
        "type": "boolean"
      "lpValue": {
        "type": "long"
      "lastRun": {
        "type": "date"
      "count": {
        "type": "long"
      "query": {
        "type": "string",
        "index": "not_analyzed"
      "action": {
        "type": "string",
        "index": "not_analyzed"
      "creator": {
        "type": "string",
        "index": "not_analyzed"
      "tags": {
        "type": "string",
        "index": "not_analyzed"
    print "Setting queries mapping\n" if ($verbose > 0);
    esPut("/${PREFIX}queries/query/_mapping?pretty&ignore_conflicts=true", $mapping);
################################################################################
sub sessionsUpdate
    my $mapping = '
  "session": {
    "_all": {"enabled": false},
    "dynamic": "true",
    "dynamic_templates": [
        "template_1": {
          "path_match": "hdrs.*",
          "match_mapping_type": "string",
          "mapping": {
            "type": "multi_field",
            "path": "full",
            "fields": {
              "snow" : {"type": "string", "analyzer" : "snowball"},
              "raw" : {"type": "string", "index" : "not_analyzed"}
        "template_georir": {
          "match_pattern": "regex",
          "path_match": ".*-(geo|rir|term)$",
          "match_mapping_type": "string",
          "mapping": {
            "omit_norms": true,
            "type": "string",
            "index": "not_analyzed"
        "template_string": {
          "match_mapping_type": "string",
          "mapping": {
            "type": "multi_field",
            "path": "full",
            "fields": {
              "snow" : {"type": "string", "analyzer" : "snowball"},
              "raw" : {"type": "string", "index" : "not_analyzed"}
    "properties": {
      "us": {
View remainder of file in raw view
FilesExpand file tree

db.pl

Latest commit

History

db.pl

File metadata and controls
