8000 moloch/db/db.pl at v0.15.0 · newdominic/moloch · GitHub
[go: up one dir, main page]
Skip to content

db.pl

Latest commit

 

History

History
executable file
·
2371 lines (2249 loc) · 66.8 KB

db.pl

File metadata and controls

executable file
·
2371 lines (2249 loc) · 66.8 KB
Raw
#!/usr/bin/perl
# This script can initialize, upgrade or provide simple maintenance for the
# moloch elastic search db
#
# Schema Versions
# 0 - Before this script existed
# 1 - First version of script; turned on strict schema; added lpms, fpms; added
# many missing items that were dynamically created by mistake
# 2 - Added email items
# 3 - Added email md5
# 4 - Added email host and ip; added help, usersimport, usersexport, wipe commands
# 5 - No schema change, new rotate command, encoding of file pos is different.
# Negative is file num, positive is file pos
# 6 - Multi fields for spi view, added xffcnt, 0.90 fixes, need to type INIT/UPGRADE
# instead of YES
# 7 - files_v3
# 8 - fileSize, memory to stats/dstats and -v flag
# 9 - http body hash, rawus
# 10 - dynamic fields for http and email headers
# 11 - Require 0.90.1, switch from soft to node, new fpd field, removed fpms field
# 12 - Added hsver, hdver fields, diskQueue, user settings, scrub* fields, user removeEnabled
# 13 - Rename rotate to expire, added smb, socks, rir fields
# 14 - New http fields, user.views
# 15 - New first byte fields, socks user
# 16 - New dynamic plugin section
# 17 - email hasheader, db,pa,by src and dst
# 18 - fields db
# 19 - users_v3
# 20 - queries
# 21 - doc_values, new tls fields, starttime/stoptime/view
# 22 - cpu to stats/dstats
# 23 - packet lengths
# 24 - field category
# 25 - cert hash
# 26 - dynamic stats, ES 2.0
# 27 - table states
use HTTP::Request::Common;
use LWP::UserAgent;
use JSON;
use Data::Dumper;
use POSIX;
use strict;
my $VERSION = 27;
my $verbose = 0;
my $PREFIX = "";
my $SHARDS = -1;
my $REPLICAS = -1;
################################################################################
sub MIN ($$) { $_[$_[0] > $_[1]] }
sub MAX ($$) { $_[$_[0] < $_[1]] }
sub commify {
scalar reverse join ',',
unpack '(A3)*',
scalar reverse shift
}
################################################################################
sub showHelp($)
{
my ($str) = @_;
print "\n", $str,"\n\n";
print "$0 [Global Options] <ESHOST:ESPORT> <command> [<command arguments>]\n";
print "\n";
print "Global Options:\n";
print " -v - Verbose, multiple increases level\n";
print " --prefix <prefix> - Prefix for table names\n";
print "\n";
print "Commands:\n";
print " init [<opts>] - Clear ALL elasticsearch moloch data and create schema\n";
print " --shards <shards> - Number of shards for sessions, default number of nodes\n";
print " --replicas <num> - Number of replicas for sessions, default 0\n";
print " upgrade [<opts>] - Upgrade Moloch's schema in 10BC8 elasticsearch from previous versions\n";
print " --shards <shards> - Number of shards for sessions, default number of nodes\n";
print " --replicas <num> - Number of replicas for sessions, default 0\n";
print " wipe - Same as init, but leaves user database untouched\n";
print " info - Information about the database\n";
print " users-export <fn> - Save the users info to <fn>\n";
print " users-import <fn> - Load the users info from <fn>\n";
print " optimize - Optimize all indices in ES\n";
print " mv <old fn> <new fn> - Move a pcap file in the database (doesn't change disk)\n";
print " rm <fn> - Remove a pcap file in the database (doesn't change disk)\n";
print " rm-missing <node> - Remove from db any MISSING file on THIS machine for named node\n";
print " rm-node <node> - Remove from db all data for node (doesn't change disk)\n";
print " add-missing <node> <dir> - Add to db any MISSING file on THIS machine for named node and directory\n";
print " expire <type> <num> [<opts>] - Perform daily maintenance and optimize all indices in ES\n";
print " type - Same as rotateIndex in ini file = hourly,daily,weekly,monthly\n";
print " num - number of indexes to keep\n";
print " --replicas <num> - Number of replicas for older sessions indices, default 0\n";
print " field disable <exp > - disable a field from being indexed\n";
print " field enable <exp > - enable a field from being indexed\n";
exit 1;
}
################################################################################
sub waitFor
{
my ($str, $help) = @_;
print "Type \"$str\" to continue - $help?\n";
while (1) {
my $answer = <STDIN>;
chomp $answer;
last if ($answer eq $str);
print "You didn't type \"$str\", for some reason you typed \"$answer\"\n";
}
}
################################################################################
sub esGet
{
my ($url, $dontcheck) = @_;
print "GET ${main::elasticsearch}$url\n" if ($verbose > 2);
my $response = $main::userAgent->get("${main::elasticsearch}$url");
if (($response->code == 500 && $ARGV[1] ne "init") || ($response->code != 200 && !$dontcheck)) {
die "Couldn't GET ${main::elasticsearch}$url the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
}
my $json = from_json($response->content);
return $json
}
################################################################################
sub esPost
{
my ($url, $content, $dontcheck) = @_;
print "POST ${main::elasticsearch}$url\n" if ($verbose > 2);
my $response = $main::userAgent->post("${main::elasticsearch}$url", Content => $content);
if ($response->code == 500 || ($response->code != 200 && $response->code != 201 && !$dontcheck)) {
die "Couldn't POST ${main::elasticsearch}$url the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
}
my $json = from_json($response->content);
return $json
}
################################################################################
sub esPut
{
my ($url, $content, $dontcheck) = @_;
print "PUT ${main::elasticsearch}$url\n" if ($verbose > 2);
my $response = $main::userAgent->request(HTTP::Request::Common::PUT("${main::elasticsearch}$url", Content => $content));
if ($response->code == 500 || ($response->code != 200 && !$dontcheck)) {
print Dumper($response);
die "Couldn't PUT ${main::elasticsearch}$url the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?\n" . $response->content;
}
my $json = from_json($response->content);
return $json
}
################################################################################
sub esDelete
{
my ($url, $dontcheck) = @_;
print "DELETE ${main::elasticsearch}$url\n" if ($verbose > 2);
my $response = $main::userAgent->request(HTTP::Request::Common::_simple_req("DELETE", "${main::elasticsearch}$url"));
if ($response->code == 500 || ($response->code != 200 && !$dontcheck)) {
die "Couldn't DELETE ${main::elasticsearch}$url the http status code is " . $response->code . " are you sure elasticsearch is running/reachable?";
}
my $json = from_json($response->content);
return $json
}
################################################################################
sub esCopy
{
my ($srci, $dsti, $type) = @_;
my $status = esGet("/_status", 1);
print "Copying " . $status->{indices}->{$PREFIX . $srci}->{docs}->{num_docs} . " elements from ${PREFIX}$srci/$type to ${PREFIX}$dsti/$type\n";
my $id = "";
while (1) {
if ($verbose > 0) {
local $| = 1;
print ".";
}
my $url;
if ($id eq "") {
$url = "/${PREFIX}$srci/$type/_search?scroll=10m&scroll_id=$id&size=500";
} else {
$url = "/_search/scroll?scroll=10m&scroll_id=$id";
}
my $incoming = esGet($url);
my $out = "";
last if (@{$incoming->{hits}->{hits}} == 0);
foreach my $hit (@{$incoming->{hits}->{hits}}) {
$out .= "{\"index\": {\"_index\": \"${PREFIX}$dsti\", \"_type\": \"$type\", \"_id\": \"" . $hit->{_id} . "\"}}\n";
$out .= to_json($hit->{_source}) . "\n";
}
$id = $incoming->{_scroll_id};
esPost("/_bulk", $out);
}
print "\n"
}
################################################################################
sub esAlias
{
my ($cmd, $index, $alias) = @_;
print "Alias cmd $cmd from $index to alias $alias\n" if ($verbose > 0);
esPost("/_aliases", '{ "actions": [ { "' . $cmd . '": { "index": "' . $PREFIX . $index . '", "alias" : "'. $PREFIX . $alias .'" } } ] }', 1);
}
################################################################################
sub tagsCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-all"
}
}';
print "Creating tags_v2 index\n" if ($verbose > 0);
esPut("/${PREFIX}tags_v2/", $settings);
esAlias("add", "tags_v2", "tags");
tagsUpdate();
}
################################################################################
sub tagsUpdate
{
my $mapping = '
{
"tag": {
"dynamic": "strict",
"properties": {
"n": {
"type": "integer"
}
}
}
}';
print "Setting tags_v2 mapping\n" if ($verbose > 0);
esPut("/${PREFIX}tags_v2/tag/_mapping", $mapping);
}
################################################################################
sub sequenceCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-all"
}
}';
print "Creating sequence index\n" if ($verbose > 0);
esPut("/${PREFIX}sequence", $settings);
sequenceUpdate();
}
################################################################################
sub sequenceUpdate
{
my $mapping = '
{
"sequence": {
"_source" : { "enabled": 0 },
"_all" : { "enabled": 0 },
"enabled" : 0
}
}';
print "Setting sequence mapping\n" if ($verbose > 0);
esPut("/${PREFIX}sequence/sequence/_mapping", $mapping);
}
################################################################################
sub filesCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 2,
"number_of_replicas": 0,
"auto_expand_replicas": "0-2"
}
}';
print "Creating files_v3 index\n" if ($verbose > 0);
esPut("/${PREFIX}files_v3", $settings);
esAlias("add", "files_v3", "files");
filesUpdate();
}
################################################################################
sub filesUpdate
{
my $mapping = '
{
"file": {
"_all": {"enabled": 0},
"_source": {"enabled": 1},
"dynamic": "strict",
"properties": {
"num": {
"type": "long",
"index": "not_analyzed"
},
"node": {
"type": "string",
"index": "not_analyzed"
},
"first": {
"type": "long",
"index": "not_analyzed"
},
"name": {
"type": "string",
"index": "not_analyzed"
},
"filesize": {
"type": "long"
},
"locked": {
"type": "short",
"index": "not_analyzed"
},
"last": {
"type": "long",
"index": "not_analyzed"
}
}
}
}';
print "Setting files_v3 mapping\n" if ($verbose > 0);
esPut("/${PREFIX}files_v3/file/_mapping", $mapping);
}
################################################################################
sub statsCreate
{
my $settings = '
{
"index": {
"store": {
"type": "memory"
}
},
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-all"
}
}';
print "Creating stats index\n" if ($verbose > 0);
esPut("/${PREFIX}stats", $settings);
statsUpdate();
}
################################################################################
sub statsUpdate
{
my $mapping = '
{
"stat": {
"_all": {enabled : false},
"_source": {enabled : true},
"dynamic": "true",
"properties": {
"hostname": {
"type": "string",
"index": "not_analyzed"
},
"currentTime": {
"type": "long"
},
"freeSpaceM": {
"type": "long",
"index": "no"
},
"totalK": {
"type": "long",
"index": "no"
},
"totalPackets": {
"type": "long",
"index": "no"
},
"monitoring": {
"type": "long",
"index": "no"
},
"totalSessions": {
"type": "long",
"index": "no"
},
"totalDropped": {
"type": "long",
"index": "no"
},
"deltaMS": {
"type": "long",
"index": "no"
},
"deltaBytes": {
"type": "long",
"index": "no"
},
"deltaPackets": {
"type": "long",
"index": "no"
},
"deltaSessions": {
"type": "long",
"index": "no"
},
"deltaDropped": {
"type": "long",
"index": "no"
},
"memory": {
"type": "long",
"index": "no"
},
"cpu": {
"type": "integer",
"index": "no"
},
"diskQueue": {
"type": "long",
"index": "no"
}
}
}
}';
print "Setting stats mapping\n" if ($verbose > 0);
esPut("/${PREFIX}stats/stat/_mapping?pretty&ignore_conflicts=true", $mapping, 1);
}
################################################################################
sub dstatsCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 2,
"number_of_replicas": 0,
"auto_expand_replicas": "0-2"
}
}';
print "Creating dstats_v1 index\n" if ($verbose > 0);
esPut("/${PREFIX}dstats_v1", $settings);
esAlias("add", "dstats_v1", "dstats");
dstatsUpdate();
}
################################################################################
sub dstatsUpdate
{
my $mapping = '
{
"dstat": {
"_all": {"enabled": false},
"_source": {"enabled": true},
"dynamic": "true",
"properties": {
"nodeName": {
"type": "string",
"index": "not_analyzed"
},
"interval": {
"type": "short"
},
"currentTime": {
"type": "long"
},
"freeSpaceM": {
"type": "long",
"index": "no"
},
"deltaMS": {
"type": "long",
"index": "no"
},
"deltaBytes": {
"type": "long",
"index": "no"
},
"deltaPackets": {
"type": "long",
"index": "no"
},
"deltaSessions": {
"type": "long",
"index": "no"
},
"deltaDropped": {
"type": "long",
"index": "no"
},
"monitoring": {
"type": "long",
"index": "no"
},
"memory": {
"type": "long",
"index": "no"
},
"cpu": {
"type": "integer",
"index": "no"
},
"diskQueue": {
"type": "long",
"index": "no"
}
}
}
}';
print "Setting dstats_v1 mapping\n" if ($verbose > 0);
esPut("/${PREFIX}dstats_v1/dstat/_mapping?pretty&ignore_conflicts=true", $mapping, 1);
}
################################################################################
sub fieldsCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-2"
}
}';
print "Creating fields index\n" if ($verbose > 0);
esPut("/${PREFIX}fields", $settings);
fieldsUpdate();
}
################################################################################
sub fieldsUpdate
{
my $mapping = '
{
"field": {
"_all": {"enabled": 0},
"_source": {"enabled": 1},
"dynamic_templates": [
{
"template_1": {
"match_mapping_type": "string",
"mapping": {
"index": "not_analyzed"
}
}
}
]
}
}';
print "Setting fields mapping\n" if ($verbose > 0);
esPut("/${PREFIX}fields/field/_mapping", $mapping);
esPost("/${PREFIX}fields/field/ip", '{
"friendlyName": "All IP fields",
"group": "general",
"help": "Search all ip fields",
"type": "ip",
"dbField": "ipall",
"portField": "portall",
"noFacet": true
}');
esPost("/${PREFIX}fields/field/port", '{
"friendlyName": "All port fields",
"group": "general",
"help": "Search all port fields",
"type": "integer",
"dbField": "portall",
"regex": "(^port\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.port$)"
}');
esPost("/${PREFIX}fields/field/rir", '{
"friendlyName": "All rir fields",
"group": "general",
"help": "Search all rir fields",
"type": "uptermfield",
"dbField": "all",
"regex": "(^rir\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.rir$)"
}');
esPost("/${PREFIX}fields/field/country", '{
"friendlyName": "All country fields",
"group": "general",
"help": "Search all country fields",
"type": "uptermfield",
"dbField": "all",
"regex": "(^country\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.country$)"
}');
esPost("/${PREFIX}fields/field/asn", '{
"friendlyName": "All ASN fields",
"group": "general",
"help": "Search all ASN fields",
"type": "textfield",
"dbField": "all",
"regex": "(^asn\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.asn$)"
}');
esPost("/${PREFIX}fields/field/host", '{
"friendlyName": "All Host fields",
"group": "general",
"help": "Search all Host fields",
"type": "lotextfield",
"dbField": "all",
"regex": "(^host\\\\.(?:(?!\\\\.cnt$).)*$|\\\\.host$)"
}');
esPost("/${PREFIX}fields/field/ip.src", '{
"friendlyName": "Src IP",
"group": "general",
"help": "Source IP",
"type": "ip",
"dbField": "a1",
"portField": "p1",
"category": "ip"
}');
esPost("/${PREFIX}fields/field/port.src", '{
"friendlyName": "Src Port",
"group": "general",
"help": "Source Port",
"type": "integer",
"dbField": "p1",
"category": "port"
}');
esPost("/${PREFIX}fields/field/asn.src", '{
"friendlyName": "Src ASN",
"group": "general",
"help": "GeoIP ASN string calculated from the source IP",
"type": "textfield",
"dbField": "as1",
"rawField": "rawas1",
"category": "asn"
}');
esPost("/${PREFIX}fields/field/country.src", '{
"friendlyName": "Src Country",
"group": "general",
"help": "Source Country",
"type": "uptermfield",
"dbField": "g1",
"category": "country"
}');
esPost("/${PREFIX}fields/field/rir.src", '{
"friendlyName": "Src RIR",
"group": "general",
"help": "Source RIR",
"type": "uptermfield",
"dbField": "rir1",
"category": "rir"
}');
esPost("/${PREFIX}fields/field/ip.dst", '{
"friendlyName": "Dst IP",
"group": "general",
"help": "Destination IP",
"type": "ip",
"dbField": "a2",
"portField": "p2",
"category": "ip"
}');
esPost("/${PREFIX}fields/field/port.dst", '{
"friendlyName": "Dst Port",
"group": "general",
"help": "Source Port",
"type": "integer",
"dbField": "p2",
"category": "port"
}');
esPost("/${PREFIX}fields/field/asn.dst", '{
"friendlyName": "Dst ASN",
"group": "general",
"help": "GeoIP ASN string calculated from the destination IP",
"type": "textfield",
"dbField": "as2",
"rawField": "rawas2",
"category": "asn"
}');
esPost("/${PREFIX}fields/field/country.dst", '{
"friendlyName": "Dst Country",
"group": "general",
"help": "Destination Country",
"type": "uptermfield",
"dbField": "g2",
"category": "country"
}');
esPost("/${PREFIX}fields/field/rir.dst", '{
"friendlyName": "Dst RIR",
"group": "general",
"help": "Destination RIR",
"type": "uptermfield",
"dbField": "rir2",
"category": "rir"
}');
esPost("/${PREFIX}fields/field/bytes", '{
"friendlyName": "Bytes",
"group": "general",
"help": "Total number of raw bytes sent AND received in a session",
"type": "integer",
"dbField": "by"
}');
esPost("/${PREFIX}fields/field/bytes.src", '{
"friendlyName": "Src Bytes",
"group": "general",
"help": "Total number of raw bytes sent by source in a session",
"type": "integer",
"dbField": "by1"
}');
esPost("/${PREFIX}fields/field/bytes.dst", '{
"friendlyName": "Dst Bytes",
"group": "general",
"help": "Total number of raw bytes sent by destination in a session",
"type": "integer",
"dbField": "by2"
}');
esPost("/${PREFIX}fields/field/databytes", '{
"friendlyName": "Data bytes",
"group": "general",
"help": "Total number of data bytes sent AND received in a session",
"type": "integer",
"dbField": "db"
}');
esPost("/${PREFIX}fields/field/databytes.src", '{
"friendlyName": "Src data bytes",
"group": "general",
"help": "Total number of data bytes sent by source in a session",
"type": "integer",
"dbField": "db1"
}');
esPost("/${PREFIX}fields/field/databytes.dst", '{
"friendlyName": "Dst data bytes",
"group": "general",
"help": "Total number of data bytes sent by destination in a session",
"type": "integer",
"dbField": "db2"
}');
esPost("/${PREFIX}fields/field/packets", '{
"friendlyName": "Packets",
"group": "general",
"help": "Total number of packets sent AND received in a session",
"type": "integer",
"dbField": "pa"
}');
esPost("/${PREFIX}fields/field/packets.src", '{
"friendlyName": "Src Packets",
"group": "general",
"help": "Total number of packets sent by source in a session",
"type": "integer",
"dbField": "pa1"
}');
esPost("/${PREFIX}fields/field/packets.dst", '{
"friendlyName": "Dst Packets",
"group": "general",
"help": "Total number of packets sent by destination in a session",
"type": "integer",
"dbField": "pa2"
}');
esPost("/${PREFIX}fields/field/ip.protocol", '{
"friendlyName": "IP Protocol",
"group": "general",
"help": "IP protocol number or friendly name",
"type": "lotermfield",
"dbField": "pr",
"transform": "ipProtocolLookup"
}');
esPost("/${PREFIX}fields/field/id", '{
"friendlyName": "Moloch ID",
"group": "general",
"help": "Moloch ID for the session",
"type": "termfield",
"dbField": "_id",
"noFacet": true
}');
esPost("/${PREFIX}fields/field/rootId", '{
"friendlyName": "Moloch Root ID",
"group": "general",
"help": "Moloch ID of the first session in a multi session stream",
"type": "termfield",
"dbField": "ro"
}');
esPost("/${PREFIX}fields/field/node", '{
"friendlyName": "Moloch Node",
"group": "general",
"help": "Moloch node name the session was recorded on",
"type": "termfield",
"dbField": "no"
}');
esPost("/${PREFIX}fields/field/file", '{
"friendlyName": "Filename",
"group": "general",
"help": "Moloch offline pcap filename",
"type": "fileand",
"dbField": "fileand"
}');
esPost("/${PREFIX}fields/field/payload8.src.hex", '{
"friendlyName": "Payload Src Hex",
"group": "general",
"help": "First 8 bytes of source payload in hex",
"type": "lotermfield",
"dbField": "fb1",
"aliases": ["payload.src"]
}');
esPost("/${PREFIX}fields/field/payload8.src.utf8", '{
"friendlyName": "Payload Src UTF8",
"group": "general",
"help": "First 8 bytes of source payload in utf8",
"type": "termfield",
"dbField": "fb1",
"transform": "utf8ToHex",
"noFacet": true
}');
esPost("/${PREFIX}fields/field/payload8.dst.hex", '{
"friendlyName": "Payload Dst Hex",
"group": "general",
"help": "First 8 bytes of destination payload in hex",
"type": "lotermfield",
"dbField": "fb2",
"aliases": ["payload.dst"]
}');
esPost("/${PREFIX}fields/field/payload8.dst.utf8", '{
"friendlyName": "Payload Dst UTF8",
"group": "general",
"help": "First 8 bytes of destination payload in utf8",
"type": "termfield",
"dbField": "fb2",
"transform": "utf8ToHex",
"noFacet": true
}');
esPost("/${PREFIX}fields/field/payload8.hex", '{
"friendlyName": "Payload Hex",
"group": "general",
"help": "First 8 bytes of payload in hex",
"type": "lotermfield",
"dbField": "fballhex",
"regex": "^payload8.(src|dst).hex$"
}');
esPost("/${PREFIX}fields/field/payload8.utf8", '{
"friendlyName": "Payload UTF8",
"group": "general",
"help": "First 8 bytes of payload in hex",
"type": "lotermfield",
"dbField": "fballutf8",
"regex": "^payload8.(src|dst).utf8$"
}');
esPost("/${PREFIX}fields/field/scrubbed.by", '{
"friendlyName": "Scrubbed By",
"group": "general",
"help": "SPI data was scrubbed by",
"type": "lotermfield",
"dbField": "scrubby"
}');
esPost("/${PREFIX}fields/field/view& BE9A quot;, '{
"friendlyName": "View Name",
"group": "general",
"help": "Moloch view name",
"type": "viewand",
"dbField": "viewand",
"noFacet": true
}');
esPost("/${PREFIX}fields/field/starttime", '{
"friendlyName": "Start Time",
"group": "general",
"help": "Session Start Time",
"type": "seconds",
"dbField": "fp"
}');
esPost("/${PREFIX}fields/field/stoptime", '{
"friendlyName": "Stop Time",
"group": "general",
"help": "Session Stop Time",
"type": "seconds",
"dbField": "lp"
}');
esPost("/${PREFIX}fields/field/dns.status/_update", '{"doc": {"type": "uptermfield"}}', 1);
esPost("/${PREFIX}fields/field/http.hasheader/_update", '{"doc": {regex: "^http.hasheader\\\\.(?:(?!\\\\.cnt$).)*$"}}', 1);
esPost("/${PREFIX}fields/field/email.subject/_update", '{"doc": {"type": "textfield"}}', 1);
}
################################################################################
sub queriesCreate
{
my $settings = '
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-2"
}
}';
print "Creating queries index\n" if ($verbose > 0);
esPut("/${PREFIX}queries", $settings);
queriesUpdate();
}
################################################################################
sub queriesUpdate
{
my $mapping = '
{
"query": {
"_all": {"enabled": 0},
"_source": {"enabled": 1},
"dynamic": "strict",
"properties": {
"name": {
"type": "string",
"index": "not_analyzed"
},
"enabled": {
"type": "boolean"
},
"lpValue": {
"type": "long"
},
"lastRun": {
"type": "date"
},
"count": {
"type": "long"
},
"query": {
"type": "string",
"index": "not_analyzed"
},
"action": {
"type": "string",
"index": "not_analyzed"
},
"creator": {
"type": "string",
"index": "not_analyzed"
},
"tags": {
"type": "string",
"index": "not_analyzed"
}
}
}
}';
print "Setting queries mapping\n" if ($verbose > 0);
esPut("/${PREFIX}queries/query/_mapping?pretty&ignore_conflicts=true", $mapping);
}
################################################################################
sub sessionsUpdate
{
my $mapping = '
{
"session": {
"_all": {"enabled": false},
"dynamic": "true",
"dynamic_templates": [
{
"template_1": {
"path_match": "hdrs.*",
"match_mapping_type": "string",
"mapping": {
"type": "multi_field",
"path": "full",
"fields": {
"snow" : {"type": "string", "analyzer" : "snowball"},
"raw" : {"type": "string", "index" : "not_analyzed"}
}
}
}
}, {
"template_georir": {
"match_pattern": "regex",
"path_match": ".*-(geo|rir|term)$",
"match_mapping_type": "string",
"mapping": {
"omit_norms": true,
"type": "string",
"index": "not_analyzed"
}
}
}, {
"template_string": {
"match_mapping_type": "string",
"mapping": {
"type": "multi_field",
"path": "full",
"fields": {
"snow" : {"type": "string", "analyzer" : "snowball"},
"raw" : {"type": "string", "index" : "not_analyzed"}
}
}
}
}
],
"properties": {
"us": {
View remainder of file in raw view
0