/* Copyright (C) 2016 NooBaa */

'use strict';

// we populate the exports object but prefer to keep name referencing

// with config.NAME so that it will be consistent with the code that imports it

// and will make searching easier.

const config = exports;

const os = require('os');

const fs = require('fs');

const path = require('path');

const assert = require('assert');

const _ = require('lodash');

const util = require('util');

const nsfs_schema_utils = require('./src/manage_nsfs/nsfs_schema_utils');

const range_utils = require('./src/util/range_utils');

const { EventEmitter } = require('events');

config.event_emitter = new EventEmitter();

/////////////////////////

// CONTAINER RESOURCES //

/////////////////////////

config.ENDPOINT_FORKS = Number(process.env.ENDPOINT_FORKS) || 0;

config.OS_MEM_RESERVED = 2 * 1024 * 1024 * 1024;

config.OS_MEM_LIMIT_TOTAL = os.totalmem() - config.OS_MEM_RESERVED;

config.OS_MEM_LIMIT = config.ENDPOINT_FORKS > 0 ? config.OS_MEM_LIMIT_TOTAL / config.ENDPOINT_FORKS : config.OS_MEM_LIMIT_TOTAL;

// The REQUEST is the "minimal" or "guaranteed" amount required for the container's operation.

// The LIMIT is the "maximal" or "burst" amount that the container can use for extended operations.

// In Kubernetes for "guaranteed" QOS class REQUEST and LIMIT both have to be provided and equal.

// This means that in most cases these configs should be the same.

// However we allow a configuration of "thin" provisioning of resources, where LIMIT is higher than REQUEST.

//

// When to use LIMIT:

// - For limiting amounts of transient memory.

// - As we do with semaphores to backpressure incoming I/O requests.

// - Suitable for memory that will be garbage collected soon enough after being processed.

// - The heap will be able to burst up and down.

//

// When to use REQUEST:

// - Less common and is currently unused

// - Might be needed when limiting pinned memory like DB caches.

config.CONTAINER_MEM_LIMIT = Number(process.env.CONTAINER_MEM_LIMIT || '') || config.OS_MEM_LIMIT;

config.CONTAINER_CPU_LIMIT = Number(process.env.CONTAINER_CPU_LIMIT || '') || os.cpus().length;

config.CONTAINER_MEM_REQUEST = Number(process.env.CONTAINER_MEM_REQUEST || '');

config.CONTAINER_CPU_REQUEST = Number(process.env.CONTAINER_CPU_REQUEST || '');

// For buffer limits we use the mem LIMIT and not REQUEST because -

// if we are in guaranteed mode then it will equal to requests anyway,

// but if limit is higher then we want to allow buffering using burst memory,

// since these buffers will get garbage collected after processing.

//

// We divide by 4 because we don't have a central buffer manager, and we need

// some fixed top limit to how much these buffers can consume from the total,

// so we just picked 4 for now.

//

// This constant is currently used for semaphores in object_io and namespace_cache.

//

// TODO we need a central buffer manager to handle all cases (other namespaces too)

config.BUFFERS_MEM_LIMIT_MIN = 32 * 1024 * 1024; // just some workable minimum size in case the container mem limit is too low.

config.BUFFERS_MEM_LIMIT_MAX = 4 * 1024 * 1024 * 1024;

config.BUFFERS_MEM_LIMIT = Math.min(

config.BUFFERS_MEM_LIMIT_MAX,

Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN,)

);

////////////////////////

// CERTIFICATE CONFIG //

////////////////////////

config.STS_SERVICE_CERT_PATH = '/etc/sts-secret';

config.S3_SERVICE_CERT_PATH = '/etc/s3-secret';

config.MGMT_SERVICE_CERT_PATH = '/etc/mgmt-secret';

config.EXTERNAL_DB_SERVICE_CERT_PATH = '/etc/external-db-secret';

//////////////////

// NODES CONFIG //

//////////////////

// TODO take nodes min and free space reserve from system/pool config

config.NODES_MIN_COUNT = 3;

config.NODES_PER_CLOUD_POOL = 1;

config.NODES_PER_MONGO_POOL = 1;

// in kubernetes use reserve of 100MB instead of 10GB

config.NODES_FREE_SPACE_RESERVE = 100 * (1024 ** 2);

// don't use agents with less than reserve + 5 GB

config.MINIMUM_AGENT_TOTAL_STORAGE = config.NODES_FREE_SPACE_RESERVE + (5 * (1024 ** 3));

config.NODE_IO_DETENTION_DISABLE = false;

config.NODE_IO_DETENTION_THRESHOLD = 60 * 1000;

config.NODE_IO_DETENTION_RECENT_ISSUES = 5;

// Picked two because minimum of nodes per pool is three

config.NODE_IO_DETENTION_TEST_NODES = 2;

config.HOSTED_AGENTS_HOST_ID = 'hosted_agents';

config.NODE_ALLOCATOR_NUM_CLUSTERS = 2;

config.AGENT_HEARTBEAT_GRACE_TIME = 10 * 60 * 1000; // grace period before an agent is considered offline

config.CLOUD_ALERT_GRACE_TIME = 3 * 60 * 1000; // grace period before dispatching alert on cloud node status

config.AGENT_RESPONSE_TIMEOUT = 1 * 60 * 1000;

config.AGENT_TEST_CONNECTION_TIMEOUT = 1 * 60 * 1000;

config.STORE_PERF_TEST_INTERVAL = 60 * 60 * 1000; // perform test_store_perf every 1 hour

config.CLOUD_MAX_ALLOWED_IO_TEST_ERRORS = 3;

config.ENABLE_DEV_RANDOM_SEED = process.env.DISABLE_DEV_RANDOM_SEED === 'false' || false;

////////////////

// RPC CONFIG //

////////////////

config.AGENT_RPC_PROTOCOL = 'n2n';

config.AGENT_RPC_PORT = process.env.AGENT_PORT || '9999';

config.RPC_CONNECT_TIMEOUT = 120 * 1000;

config.RPC_SEND_TIMEOUT = 120 * 1000;

config.RPC_PING_INTERVAL_MS = 20000;

// setting number of pings above the time it takes to get connect timeout

config.RPC_PING_EXHAUSTED_COUNT = (config.RPC_CONNECT_TIMEOUT / config.RPC_PING_INTERVAL_MS) + 2;

config.RECONN_BACKOFF_BASE = 250;

config.RECONN_BACKOFF_MAX = 5000;

config.RECONN_BACKOFF_FACTOR = 1.2;

config.CLOUD_AGENTS_N2N_PORT = 60100;

// TODO: Should check what PORT we are interested in taking

config.MONGO_AGENTS_N2N_PORT = 60100;

config.N2N_OFFER_INTERNAL = false;

/////////////////////

// ENDPOINT CONFIG //

/////////////////////

config.AMZ_DATE_MAX_TIME_SKEW_MILLIS = 15 * 60 * 1000;

config.ENDPOINT_MONITOR_INTERVAL = 10 * 60 * 1000; // 10min

// Keep connection on long requests

config.S3_KEEP_ALIVE_WHITESPACE_INTERVAL = 15 * 1000;

config.S3_MD_SIZE_LIMIT = 2 * 1024;

// Semaphore monitoring execution interval

config.SEMAPHORE_MONITOR_DELAY = 10 * 1000;

// Semaphore metrics average calculation intervals in minutes, values need to be in ascending order

config.SEMAPHORE_METRICS_AVERAGE_INTERVALS = Object.freeze(['1', '5', '10']);

config.ENABLE_SEMAPHORE_MONITOR = true;

config.ENABLE_OBJECT_IO_SEMAPHORE_MONITOR = true;

config.ENDPOINT_HTTP_SERVER_REQUEST_TIMEOUT = 300 * 1000;

config.ENDPOINT_HTTP_SERVER_KEEPALIVE_TIMEOUT = 5 * 1000;

config.ENDPOINT_HTTP_MAX_REQUESTS_PER_SOCKET = 0; // 0 = no limit

// For now we enable fixed CORS for all buckets

// but this should become a setting per bucket which is configurable

// with the s3 put-bucket-cors api.

// note that browsers do not really allow origin=* with credentials,

// but we just allow both from our side for simplicity.

config.S3_CORS_ENABLED = true;

config.S3_CORS_ALLOW_ORIGIN = '*';

config.S3_CORS_ALLOW_CREDENTIAL = 'true';

config.S3_CORS_ALLOW_METHODS = [

'GET',

'POST',

'PUT',

'DELETE',

'OPTIONS'

].join(',');

config.S3_CORS_ALLOW_HEADERS = [

'Content-Type',

'Content-MD5',

'Authorization',

'X-Amz-User-Agent',

'X-Amz-Date',

'ETag',

'X-Amz-Content-Sha256',

'amz-sdk-invocation-id',

'amz-sdk-request',

].join(',');

config.S3_CORS_EXPOSE_HEADERS = [

'ETag',

'X-Amz-Version-Id'

].join(',');

config.STS_CORS_EXPOSE_HEADERS = 'ETag';

config.DENY_UPLOAD_TO_STORAGE_CLASS_STANDARD = false;

// S3_RESTORE_MAX_DAYS controls that for how many maximum number

// of days an object can be restored using `restore-object` call.

config.S3_RESTORE_REQUEST_MAX_DAYS = 30;

/**

* S3_RESTORE_MAX_DAYS_BEHAVIOUR controls whether to truncate the

* requested number of days in restore request or whether to deny the request.

* @type {'DENY' | 'TRUNCATE'}

*/

config.S3_RESTORE_REQUEST_MAX_DAYS_BEHAVIOUR = 'TRUNCATE';

/**

* S3_MAX_KEY_LENGTH controls the maximum key length that will be accepted

* by NooBaa endpoints.

*

* This value is 1024 bytes for S3 but the default is `Infinity`

*/

config.S3_MAX_KEY_LENGTH = Infinity;

/**

* S3_MAX_BUCKET_NAME_LENGTH controls the maximum bucket name length that

* will be accepted by NooBaa endpoints.

*

* This value is 63 bytes for S3 but the default is `Infinity`

*/

config.S3_MAX_BUCKET_NAME_LENGTH = Infinity;

/////////////////////

// SECRETS CONFIG //

/////////////////////

if (process.env.CONTAINER_PLATFORM || process.env.LOCAL_MD_SERVER) {

config.JWT_SECRET = process.env.JWT_SECRET || _get_data_from_file(`/etc/noobaa-server/jwt`);

config.SERVER_SECRET = process.env.SERVER_SECRET || _get_data_from_file(`/etc/noobaa-server/server_secret`);

config.NOOBAA_AUTH_TOKEN = process.env.NOOBAA_AUTH_TOKEN || _get_data_from_file(`/etc/noobaa-auth-token/auth_token`);

}

config.ROOT_KEY_MOUNT = '/etc/noobaa-server/root_keys';

///////////////

// DB CONFIG //

///////////////

config.DB_TYPE = /** @type {nb.DBType} */ (process.env.DB_TYPE || 'postgres');

config.POSTGRES_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 80 : 10;

///////////////////

// SYSTEM CONFIG //

///////////////////

config.DEFAULT_POOL_TYPE = 'INTERNAL'; // use 'HOSTS' for setting up a pool of FS backingstores instead

config.DEFAULT_POOL_NAME = 'backingstores'; // only used when config.DEFAULT_POOL_TYPE = 'HOSTS'

config.DEFAULT_BUCKET_NAME = 'first.bucket';

config.INTERNAL_STORAGE_POOL_NAME = 'system-internal-storage-pool';

// config.SPILLOVER_TIER_NAME = 'bucket-spillover-tier';

config.ALLOW_BUCKET_CREATE_ON_INTERNAL = true;

config.BUCKET_AUTOCONF_TIER2_ENABLED = false;

//////////////////////////

// MD AGGREGATOR CONFIG //

//////////////////////////

config.MD_AGGREGATOR_ENABLED = true;

config.MD_AGGREGATOR_INTERVAL = 30000;

// the max cycles limits how many intervals the aggregator will split the gap

// the higher this value the higher the time intervals it will scan when trying to close time gaps

config.MD_AGGREGATOR_MAX_CYCLES = 2880; // 1 day / 30 seconds

// Currently the grace is 3 cycles of md_aggregator

// This grace is used since we can hold up an ObjectID and not push it inside the DB

// Which will mean that it will be pushed later on with a previous date

config.MD_GRACE_IN_MILLISECONDS = config.MD_AGGREGATOR_INTERVAL * 3;

config.MD_AGGREGATOR_BATCH = 100;

// Date was chosen as default NooBaa epoch date 2015

config.NOOBAA_EPOCH = 1430006400000;

///////////////

// IO CONFIG //

///////////////

config.MAX_OBJECT_PART_SIZE = 64 * 1024 * 1024;

config.IO_CHUNK_READ_CACHE_SIZE = 256 * 1024 * 1024;

config.ALLOCATE_RETRY_DELAY_MS = 500;

config.IO_WRITE_BLOCK_RETRIES = 5;

config.IO_WRITE_BLOCK_TIMEOUT = 120 * 1000;

config.IO_WRITE_RETRY_DELAY_MS = 100;

config.IO_REPLICATE_BLOCK_RETRIES = 3;

config.IO_REPLICATE_BLOCK_TIMEOUT = 120 * 1000;

config.IO_REPLICATE_RETRY_DELAY_MS = 100;

config.IO_READ_BLOCK_TIMEOUT = 120 * 1000;

config.IO_DELETE_BLOCK_TIMEOUT = 120 * 1000;

config.IO_WRITE_PART_ATTEMPTS_EXHAUSTED = 120 * 1000;

config.IO_WRITE_CONCURRENCY_GLOBAL = 256;

config.IO_REPLICATE_CONCURRENCY_GLOBAL = 256;

config.IO_READ_CONCURRENCY_GLOBAL = 256;

config.IO_WRITE_CONCURRENCY_AGENT = 16;

config.IO_WRITE_CONCURRENCY_AGENT_CLOUD = 128;

config.IO_REPLICATE_CONCURRENCY_AGENT = 16;

config.IO_READ_CONCURRENCY_AGENT = 16;

config.IO_READ_RANGE_CONCURRENCY = 32;

config.IO_STREAM_SPLIT_SIZE = 32 * 1024 * 1024;

// This is the maximum IO memory usage cap inside single semaphore job

config.IO_STREAM_SEMAPHORE_SIZE_CAP = config.IO_STREAM_SPLIT_SIZE * 8;

config.IO_STREAM_MINIMAL_SIZE_LOCK = 1 * 1024 * 1024;

config.IO_OBJECT_RANGE_ALIGN = 32 * 1024 * 1024;

config.IO_STREAM_SEMAPHORE_TIMEOUT = 120 * 1000;

config.VIDEO_READ_STREAM_PRE_FETCH_LOAD_CAP = 5 * 1000;

config.IO_SEMAPHORE_CAP = Math.max(

config.BUFFERS_MEM_LIMIT, // upper limit

config.IO_STREAM_SEMAPHORE_SIZE_CAP, // minimal size needed to complete jobs in object_io

);

config.DEDUP_ENABLED = true;

config.IO_CALC_MD5_ENABLED = true;

config.IO_CALC_SHA256_ENABLED = true;

config.ERROR_INJECTON_ON_WRITE = 0;

config.ERROR_INJECTON_ON_READ = 0;

///////////////////////////

// AGENT BLOCKS VERIFIER //

///////////////////////////

config.AGENT_BLOCKS_VERIFIER_ENABLED = false;

// TODO: Should check what is the optiomal amount of batch

config.AGENT_BLOCKS_VERIFIER_BATCH_SIZE = 1000;

config.AGENT_BLOCKS_VERIFIER_BATCH_DELAY = 50;

config.AGENT_BLOCKS_VERIFIER_ERROR_DELAY = 3000;

config.AGENT_BLOCKS_VERIFIER_RESTART_DELAY = 30000;

config.AGENT_BLOCKS_VERIFIER_TIMEOUT = 120 * 1000;

////////////////////////////

// AGENT BLOCKS RECLAIMER //

////////////////////////////

config.AGENT_BLOCKS_RECLAIMER_ENABLED = true;

// TODO: Should check what is the optiomal amount of batch

config.AGENT_BLOCKS_RECLAIMER_BATCH_SIZE = 1000;

config.AGENT_BLOCKS_RECLAIMER_BATCH_DELAY = 50;

config.AGENT_BLOCKS_RECLAIMER_ERROR_DELAY = 3000;

config.AGENT_BLOCKS_RECLAIMER_RESTART_DELAY = 30000;

////////////////////

// REBUILD CONFIG //

////////////////////

config.REBUILD_NODE_ENABLED = true;

config.REBUILD_NODE_CONCURRENCY = 3;

config.REBUILD_NODE_OFFLINE_GRACE = 5 * 60000;

config.REBUILD_NODE_BATCH_SIZE = 10;

config.REBUILD_NODE_BATCH_DELAY = 50;

// TODO: Temporary using the same number but later on they will be different cellings

config.MIN_TIER_FREE_THRESHOLD = 100 * 1024 * 1024;

config.MAX_TIER_FREE_THRESHOLD = 100 * 1024 * 1024;

config.ENOUGH_ROOM_IN_TIER_THRESHOLD = 200 * 1024 * 1024;

config.CHUNK_MOVE_LIMIT = 10;

config.SCRUBBER_ENABLED = true;

config.SCRUBBER_BATCH_SIZE = 10;

config.SCRUBBER_BATCH_DELAY = 50;

config.SCRUBBER_ERROR_DELAY = 3000;

config.SCRUBBER_RESTART_DELAY = 30000;

config.MIRROR_WRITER_ENABLED = true;

config.MIRROR_WRITER_BATCH_SIZE = 10;

config.MIRROR_WRITER_BATCH_DELAY = 50;

config.MIRROR_WRITER_ERROR_DELAY = 3000;

config.MIRROR_WRITER_EMPTY_DELAY = 30000;

config.MIRROR_WRITER_MARKER_STORE_PERIOD = 10 * 60000; // store markers every 10 min

config.TIER_TTF_WORKER_ENABLED = true;

config.TIER_TTF_WORKER_BATCH_DELAY = 500;

config.TIER_TTF_WORKER_EMPTY_DELAY = 30000;

config.TIER_SPILLBACK_WORKER_ENABLED = true;

config.TIER_SPILLBACK_BATCH_SIZE = 50;

config.TIER_SPILLBACK_BATCH_DELAY = 50;

config.TIER_SPILLBACK_EMPTY_DELAY = 30000;

config.TIER_SPILLBACK_MIN_FREE = config.MIN_TIER_FREE_THRESHOLD * 2;

config.BUCKET_RECLAIMER_ENABLED = true;

config.BUCKET_RECLAIMER_EMPTY_DELAY = 30000;

config.BUCKET_RECLAIMER_BATCH_SIZE = 1000;

config.BUCKET_RECLAIMER_BATCH_DELAY = 100;

config.BUCKET_RECLAIMER_ERROR_DELAY = 3000;

config.OBJECT_RECLAIMER_ENABLED = true;

config.OBJECT_RECLAIMER_EMPTY_DELAY = 30000;

config.OBJECT_RECLAIMER_BATCH_SIZE = 100;

config.OBJECT_RECLAIMER_BATCH_DELAY = 100;

config.OBJECT_RECLAIMER_ERROR_DELAY = 3000;

//////////////////

// CHUNK CONFIG //

//////////////////

// SPLIT

config.CHUNK_SPLIT_AVG_CHUNK = 4 * 1024 * 1024;

config.CHUNK_SPLIT_DELTA_CHUNK = config.CHUNK_SPLIT_AVG_CHUNK / 4;

// CODER

config.CHUNK_CODER_DIGEST_TYPE = 'sha384';

config.CHUNK_CODER_FRAG_DIGEST_TYPE = 'sha1';

config.CHUNK_CODER_COMPRESS_TYPE = process.env.NOOBAA_DISABLE_COMPRESSION === 'true' ? undefined : 'snappy';

config.CHUNK_CODER_CIPHER_TYPE = 'aes-256-gcm';

// ERASURE CODES

config.CHUNK_CODER_REPLICAS = 1;

config.CHUNK_CODER_EC_DATA_FRAGS = 4;

config.CHUNK_CODER_EC_PARITY_FRAGS = 2;

config.CHUNK_CODER_EC_PARITY_TYPE = 'cm256';

config.CHUNK_CODER_EC_TOLERANCE_THRESHOLD = 2;

config.CHUNK_CODER_EC_IS_DEFAULT = false;

// DEDUP

config.MIN_CHUNK_AGE_FOR_DEDUP = 60 * 60 * 1000; // 1 hour

//////////////////////////

// DEDUP INDEXER CONFIG //

//////////////////////////

config.DEDUP_INDEXER_ENABLED = true;

config.DEDUP_INDEXER_BATCH_SIZE = 200;

config.DEDUP_INDEXER_BATCH_DELAY = 1000;

config.DEDUP_INDEXER_ERROR_DELAY = 10 * 1000;

config.DEDUP_INDEXER_RESTART_DELAY = 12 * 60 * 60 * 1000; //12h

config.DEDUP_INDEXER_IGNORE_BACK_TIME = 7 * 24 * 60 * 60 * 1000; // dedup indexer will ignore dedup keys from the last week

config.DEDUP_INDEXER_LOW_KEYS_NUMBER = 8 * 1000000; // 8M keys max in index ~ 1.5GB

config.DEDUP_INDEXER_CHECK_INDEX_CYCLE = 60000;

///////////////////////

// DB CLEANER CONFIG //

///////////////////////

config.DB_CLEANER_ENABLED = true;

config.DB_CLEANER_CYCLE = 2 * 60 * 60 * 1000; // 2 hours

config.DB_CLEANER_BACK_TIME = 3 * 30 * 24 * 60 * 60 * 1000; // 3 months

config.DB_CLEANER_DOCS_LIMIT = 1000;

config.DB_CLEANER_MAX_TOTAL_DOCS = 10000;

/////////////////////

// CLOUD RESOURCES //

/////////////////////

// This parameter is used to toggle pool deletion checks that enforce that only the pool and system owners can delete the pool

config.RESTRICT_RESOURCE_DELETION = process.env.RESTRICT_RESOURCE_DELETION === 'true';

// EXPERIMENTAL!! switch to turn off the use of signed urls to delegate cloud read\writes to object_io (s3 compatible only)

// when this is set to true, block_store_s3 will return s3 credentials to the block_store_client so it can

// connect directly to the cloud target (and not via a signed url)

config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_DELEGATION = {

DEFAULT: false,

FLASHBLADE: true,

IBM_COS: true,

};

config.EXPERIMENTAL_DISABLE_S3_COMPATIBLE_METADATA = {

DEFAULT: false,

FLASHBLADE: true,

S3_COMPATIBLE: true,

IBM_COS: true,

};

config.DEFAULT_S3_AUTH_METHOD = {

AWS: 'AWS_V4',

FLASHBLADE: 'AWS_V4',

IBM_COS: 'AWS_V2',

};

//////////////////////

// LIFECYCLE CONFIG //

//////////////////////

config.LIFECYCLE_INTERVAL = 8 * 60 * 60 * 1000; // 8h

config.LIFECYCLE_BATCH_SIZE = 1000;

config.LIFECYCLE_SCHEDULE_MIN = 5 * 1000 * 60; // run every 5 minutes

config.LIFECYCLE_ENABLED = true;

//////////////////////////

// STATISTICS_COLLECTOR //

/////////////////////////

config.STATISTICS_COLLECTOR_ENABLED = true;

config.STATISTICS_COLLECTOR_INTERVAL = 6 * 60 * 60 * 1000; // 6 hours

config.STATISTICS_COLLECTOR_EXPIRATION = 31 * 24 * 60 * 60 * 1000; // 1 month

///////////////////

// USAGE REPORTS //

///////////////////

config.SERVICES_TYPES = Object.freeze(['AWS', 'AWSSTS', 'AZURE', 'S3_COMPATIBLE', 'GOOGLE', 'FLASHBLADE', 'IBM_COS']);

config.USAGE_AGGREGATOR_INTERVAL = 24 * 60 * 60 * 1000; // 24 hours

config.BLOCK_STORE_USAGE_INTERVAL = 1 * 60 * 1000; // 1 minute

//////////////////

// DEBUG CONFIG //

//////////////////

config.DEBUG_MODE_PERIOD = 10 * 60 * 1000; // 10 minutes for increased debug level

config.dbg_log_level = 0;

config.DEBUG_FACILITY = 'LOG_LOCAL0';

config.EVENT_FACILITY = 'LOG_LOCAL2';

config.EVENT_LOGGING_ENABLED = true;

config.EVENT_LEVEL = 5;

config.LOG_TO_STDERR_ENABLED = true;

config.LOG_TO_SYSLOG_ENABLED = false;

config.LOG_COLOR_ENABLED = process.env.NOOBAA_LOG_COLOR ? process.env.NOOBAA_LOG_COLOR === 'true' : true;

// TEST Mode

config.test_mode = false;

// On Premise NVA params

config.on_premise = {

base_url: "https://s3-eu-west-1.amazonaws.com/noobaa-download/on_premise/v_",

nva_part: "NVA_Upgrade.tgz"

};

// the threshold in ms for logging long running queries

config.LONG_DB_QUERY_THRESHOLD = parseInt(process.env.LONG_DB_QUERY_THRESHOLD, 10) || 5000;

config.INVALID_SCHEMA_DB_INSPECT_ENABLED = true;

/*

Central Stats Collection & Diagnostics

*/

const is_windows = (process.platform === 'win32');

if (!is_windows) {

process.env.ProgramData = '/tmp';

}

config.PHONE_HOME_BASE_URL = 'https://phonehome.noobaa.com';

config.central_stats = {

send_stats: false,

partial_send_time_cycle: 5 * 60 * 1000, //5 min

full_cycle_ratio: 6, // One full cycle per 5 partial cycles

previous_diag_packs_dir: process.env.ProgramData + '/prev_diags',

previous_diag_packs_count: 2 //TODO: We might want to split between agent and server

};

/*

Clustering Defaults

*/

config.MONGO_DEFAULTS = {

CFG_DB_PATH: '/data/mongo/cluster/cfg0',

CFG_PORT: '26050',

CFG_RSET_NAME: 'config0',

SHARD_SRV_PORT: '27000',

COMMON_PATH: '/data/mongo/cluster',

CONNECT_RETRY_INTERVAL: 3 * 1000,

CONNECT_MAX_WAIT: 5 * 60 * 1000,

ROOT_CA_PATH: '/data/mongo/ssl/root-ca.pem',

SERVER_CERT_PATH: '/data/mongo/ssl/server.pem',

CLIENT_CERT_PATH: '/data/mongo/ssl/client.pem',

};

config.CLUSTERING_PATHS = {

SUPER_FILE: '/data/noobaa_supervisor.conf',

};

config.NAMED_DEFAULTS = {

FORWARDERS_OPTION_FILE: '/etc/noobaa_configured_dns.conf'

};

config.CLUSTER_HB_INTERVAL = 1 * 60000;

config.CLUSTER_MASTER_INTERVAL = 10000;

config.CLUSTER_NODE_MISSING_TIME = 3 * 60000;

config.SUPERVISOR_PROGRAM_SEPERATOR = '#endprogram';

config.SUPERVISOR_DEFAULTS = {

STOPSIGNAL: 'KILL',

DIRECTORY: '/root/node_modules/noobaa-core'

};

config.SERVER_MIN_REQUIREMENTS = {

// Should be synced with the container requirements

RAM_GB: 2,

STORAGE_GB: 50,

CPU_COUNT: 1

};

// we currently use ~600MB during the upgrade process. use twice as much as a limit

config.MIN_MEMORY_FOR_UPGRADE = 1200 * 1024 * 1024;

//////////////////////////////

// ALERTING & EVENTS CONFIG //

//////////////////////////////

config.SEND_EVENTS_REMOTESYS = true;

config.PROMETHEUS_ENABLED = true;

config.PROMETHEUS_SERVER_RETRY_COUNT = Infinity;

config.PROMETHEUS_SERVER_RETRY_DELAY = 5 * 60 * 1000; // 5 min

config.PROMETHEUS_PREFIX = 'NooBaa_';

// Ports where prometheus metrics are served

config.WS_METRICS_SERVER_PORT = 7001;

config.BG_METRICS_SERVER_PORT = 7002;

config.HA_METRICS_SERVER_PORT = 7003;

config.EP_METRICS_SERVER_PORT = 7004;

//////////////////////////////

// OAUTH RELATES //

//////////////////////////////

config.OAUTH_REDIRECT_ENDPOINT = 'fe/oauth/callback';

config.OAUTH_REQUIRED_SCOPE = 'user:info';

config.OAUTH_REQUIRED_GROUPS = [

'system:cluster-admins',

'cluster-admins'

];

//////////////////////////////

// KUBERNETES RELATES //

//////////////////////////////

config.KUBE_APP_LABEL = 'noobaa';

config.KUBE_SA_TOKEN_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token';

config.KUBE_NAMESPACE_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace';

config.KUBE_API_ENDPOINTS = {

TOKEN_REVIEW: '/apis/authentication.k8s.io/v1/tokenreviews'

};

//////////////////////////////

// ACCOUNT PREFERENCES //

//////////////////////////////

config.DEFAULT_ACCOUNT_PREFERENCES = {

ui_theme: 'DARK'

};

///////////////////////////////////

// REMOTE NOOBAA DEPLOYMENT INFO //

///////////////////////////////////

config.REMOTE_NOOAA_NAMESPACE = `remote-${config.KUBE_APP_LABEL}`;

///////////////////////////////

// FILES RELATED //

///////////////////////////////

config.INLINE_MAX_SIZE = 4096;

// Object SDK bucket cache expiration time

config.OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS = 60000;

//////////////////////////////

// OPERATOR RELATED //

//////////////////////////////

config.OPERATOR_ACCOUNT_EMAIL = 'operator@noobaa.io';

///////////////////////////////

// WORM RELATED //

///////////////////////////////

config.WORM_ENABLED = false;

////////////////////////////////

// NAMESPACE MONITOR //

////////////////////////////////

config.NAMESPACE_MONITOR_ENABLED = true;

config.NAMESPACE_MONITOR_DELAY = 3 * 60 * 1000;

//////////////////////////////////

// NAMESPACE MODE CALC //

//////////////////////////////////

config.NS_MAX_ALLOWED_IO_ERRORS = 9;

////////////////////////////////

// BUCKET REPLICATOR //

////////////////////////////////

config.BUCKET_REPLICATOR_DELAY = 5 * 60 * 1000;

config.BUCKET_REPLICATOR_LIST_LIMIT = 1000;

config.BUCKET_REPLICATION_MAX_RULES = 1000;

config.BUCKET_REPLICATION_MAX_DST_BUCKETS = 100;

config.REPLICATION_SEMAPHORE_CAP = 5000;

config.REPLICATION_SEMAPHORE_TIMEOUT = 300 * 1000;

config.REPLICATION_ENABLED = true;

config.LOG_REPLICATION_ENABLED = true;

config.AWS_LOG_CANDIDATES_LIMIT = 10;

config.BUCKET_LOG_REPLICATOR_DELAY = 5 * 60 * 1000;

config.AZURE_QUERY_TRUNCATION_MAX_SIZE_IN_BITS = 10 * 1024 * 1024;

config.BUCKET_DIFF_FOR_REPLICATION = true;

////////////////////////////////

// BUCKET LOGGING //

////////////////////////////////

config.BUCKET_LOG_UPLOAD_ENABLED = true;

config.BUCKET_LOG_UPLOADER_DELAY = 5 * 60 * 1000;

config.BUCKET_LOG_TYPE = process.env.GUARANTEED_LOGS_PATH ? 'PERSISTENT' : 'BEST_EFFORT';

config.PERSISTENT_BUCKET_LOG_DIR = process.env.GUARANTEED_LOGS_PATH;

config.PERSISTENT_BUCKET_LOG_NS = 'bucket_logging';

config.BUCKET_LOG_CONCURRENCY = 10;

////////////////////////////////

// NOTIFICATIONS //

////////////////////////////////

config.NOTIFICATION_LOG_NS = 'notification_logging';

config.NOTIFICATION_LOG_DIR = process.env.NOTIFICATION_LOG_DIR;

///////////////////////////

// KEY ROTATOR //

///////////////////////////

config.KEY_ROTATOR_ENABLED = true;

config.KEY_ROTATOR_RUN_INTERVAL = 24 * 60 * 60 * 1000; // Once a day,

config.KEY_ROTATOR_ERROR_DELAY = 10 * 60 * 1000; // Run again in 10 minutes

///////////////////////

// NAMESPACE CACHING //

///////////////////////

config.NAMESPACE_CACHING = {

DEFAULT_CACHE_TTL_MS: 0,

DEFAULT_BLOCK_SIZE: 64 * 1024,

DEFAULT_MAX_CACHE_OBJECT_SIZE: 4 * 1024 * 1024 * 1024 * 1024,

DISABLE_BUCKET_FREE_SPACE_CHECK: false,

CACHE_USAGE_PERCENTAGE_HIGH_THRESHOLD: 80,

PART_COUNT_HIGH_THRESHOLD: 5,

CACHED_PERCENTAGE_LOW_THRESHOLD: 40,

CACHED_PERCENTAGE_HIGH_THRESHOLD: 80,

UPLOAD_SEMAPHORE_TIMEOUT: 30 * 1000,

MIN_OBJECT_AGE_FOR_GC: 1000 * 60 * 60 * 24,

UPLOAD_SEMAPHORE_CAP: config.BUFFERS_MEM_LIMIT,

ACL_HANDLING: /** @type { 'reject' | 'pass-through' | '' } */ (''),

};

assert(config.NAMESPACE_CACHING.DEFAULT_BLOCK_SIZE <= config.NAMESPACE_CACHING.DEFAULT_MAX_CACHE_OBJECT_SIZE);

assert(config.NAMESPACE_CACHING.DEFAULT_BLOCK_SIZE <= config.MAX_OBJECT_PART_SIZE);

assert(config.NAMESPACE_CACHING.DEFAULT_BLOCK_SIZE > config.INLINE_MAX_SIZE);

//////////////////

// NAMESPACE FS //

//////////////////

config.NSFS_BUF_SIZE_XS = 4 * 1024;

config.NSFS_BUF_SIZE_S = 64 * 1024;

config.NSFS_BUF_SIZE_M = 1 * 1024 * 1024;

config.NSFS_BUF_SIZE_L = 8 * 1024 * 1024;

// This configs help calculate the number of small and XS buffers that will be created

// The top number of buffers we want of the small and extra small sizes - 512 seems to be enough

config.NSFS_WANTED_BUFFERS_NUMBER = 512;

// The maximum size that the total XS buffers will take - as XS should be few kb 8M should be enough

config.NSFS_MAX_MEM_SIZE_XS = 8 * 1024 * 1024;

// The maximum size that the total small buffers will take - as S should be tens of kb 32M should be enough

config.NSFS_MAX_MEM_SIZE_S = 32 * 1024 * 1024;

// Semaphore size will give the amount of XS buffers that fits in 8MB up to 512 buffers

config.NSFS_BUF_POOL_MEM_LIMIT_XS = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_XS / config.NSFS_BUF_SIZE_XS),

config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_XS;

// Semaphore size will give the amount of small buffers that fits in 32MB up to 512 buffers

config.NSFS_BUF_POOL_MEM_LIMIT_S = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_S / config.NSFS_BUF_SIZE_S),

config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_S;

// Semaphore size will give 90% of remainning memory to large buffer size, 10% to medium

config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down((config.BUFFERS_MEM_LIMIT -

config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1,

config.NSFS_BUF_SIZE_M);

config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down((config.BUFFERS_MEM_LIMIT -

config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9,

config.NSFS_BUF_SIZE_L);

config.NSFS_BUF_WARMUP_SPARSE_FILE_READS = true;

config.NSFS_DEFAULT_IOV_MAX = 1024; // see IOV_MAX in https://man7.org/linux/man-pages/man0/limits.h.0p.html

// the temporary path for uploads and other internal files

config.NSFS_TEMP_DIR_NAME = '.noobaa-nsfs';

config.NSFS_FOLDER_OBJECT_NAME = '.folder';

config.NSFS_DIR_CACHE_MAX_DIR_SIZE = 64 * 1024 * 1024;

config.NSFS_DIR_CACHE_MIN_DIR_SIZE = 64;

config.NSFS_DIR_CACHE_MAX_TOTAL_SIZE = 4 * config.NSFS_DIR_CACHE_MAX_DIR_SIZE;

config.NSFS_OPEN_READ_MODE = 'r'; // use 'rd' for direct io

config.BASE_MODE_FILE = 0o666;

config.BASE_MODE_DIR = 0o777;

// umask that will be used for creation of directories and files with no permissions for 'others'

config.NSFS_UMASK = 0o007;

// The threshold in milliseconds for prompting a warning

config.NSFS_WARN_THRESHOLD_MS = 100;

config.NSFS_CALCULATE_MD5 = false;

config.NSFS_TRIGGER_FSYNC = true;

config.NSFS_CHECK_BUCKET_BOUNDARIES = true;

config.NSFS_REMOVE_PARTS_ON_COMPLETE = true;

config.NSFS_BUF_POOL_WARNING_TIMEOUT = 2 * 60 * 1000;

config.NSFS_SEM_WARNING_TIMEOUT = 10 * 60 * 1000;

// number of rename retries in case of deleted destination directory

config.NSFS_RENAME_RETRIES = 10;

config.NSFS_MKDIR_PATH_RETRIES = 3;

config.NSFS_RANDOM_DELAY_BASE = 70;

config.NSFS_VERSIONING_ENABLED = true;

config.NSFS_UPDATE_ISSUES_REPORT_ENABLED = true;

config.NSFS_EXIT_EVENTS_TIME_FRAME_MIN = 24 * 60; // per day

config.NSFS_MAX_EXIT_EVENTS_PER_TIME_FRAME = 10; // allow max 10 failed forks per day

config.NSFS_GLACIER_LOGS_DIR = '/var/run/noobaa-nsfs/wal';

config.NSFS_GLACIER_LOGS_POLL_INTERVAL = 10 * 1000;

// NSFS_GLACIER_ENABLED can override internal autodetection and will force

// the use of restore for all objects.

config.NSFS_GLACIER_ENABLED = false;

config.NSFS_GLACIER_LOGS_ENABLED = false;

config.NSFS_GLACIER_BACKEND = 'TAPECLOUD';

// TAPECLOUD Glacier backend specific configs

config.NSFS_GLACIER_TAPECLOUD_BIN_DIR = '/opt/ibm/tapecloud/bin';

// If set to true will disable cleanup of the task show output

// Should be used only for debugging or else will keep filling

// up the disk space.

config.NSFS_GLACIER_TAPECLOUD_PRESERVE_TASK_SHOW_OUTPUT = false;

// NSFS_GLACIER_MIGRATE_INTERVAL indicates the interval between runs

// of `manage_nsfs glacier migrate`

config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;

// NSFS_GLACIER_RESTORE_INTERVAL indicates the interval between runs

// of `manage_nsfs glacier restore`

config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;

// NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies

// when NooBaa should allow running glacier expiry process

// NOTE: This will also be in the same timezone as specified in

// NSFS_GLACIER_EXPIRY_TZ

config.NSFS_GLACIER_EXPIRY_RUN_TIME = '03:00';

// NSFS_GLACIER_EXPIRY_RUN_TIME_TOLERANCE_MINS configures the delay

// tolerance in minutes.

//

// eg. If the expiry run time is set to 03:00 and the tolerance is

// set to be 2 mins then the expiry can trigger till 03:02 (unless

// already triggered between 03:00 - 03:02

config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS = 2 * 60;

/** @type {'UTC' | 'LOCAL'} */

config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL';

// Format must be HH:MM:SS

//

// if set to empty string then time of processing

// the request will be used

config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY = '';

config.NSFS_STATFS_CACHE_SIZE = 10000;

config.NSFS_STATFS_CACHE_EXPIRY_MS = 1 * 1000;

// NSFS_LOW_FREE_SPACE_CHECK_ENABLED if set to true will use the below mentioned

// thresholds to determine if the writes should be denied even

// before we hit ENOSPC more filesystem.

config.NSFS_LOW_FREE_SPACE_CHECK_ENABLED = false;

// NSFS_LOW_FREE_SPACE_MB controls that how much space in

// bytes does NooBaa consider to be too low to perform `PUT` operations

// safely.

config.NSFS_LOW_FREE_SPACE_MB = 8 * 1024;

// NSFS_LOW_FREE_SPACE_PERCENT controls how much space in terms of

// percentage does NooBaa consider to be too low to perform `PUT`

// operations safely.

config.NSFS_LOW_FREE_SPACE_PERCENT = 0.08;

// NSFS_LOW_FREE_SPACE_MB_UNLEASH controls how much much space in bytes

// does NooBaa consider to be enough to perform `PUT` operations

// safely.

config.NSFS_LOW_FREE_SPACE_MB_UNLEASH = 10 * 1024;

// NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH controls how much much space in of

// percentage does NooBaa consider to be enough to perform `PUT`

// operations safely.

config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10;

// anonymous account name

config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';

////////////////////////////

// NSFS NON CONTAINERIZED //

////////////////////////////

config.ENDPOINT_PROCESS_TITLE = 'noobaa';

config.NC_RELOAD_CONFIG_INTERVAL = 10 * 1000;

config.NSFS_NC_CONF_DIR_REDIRECT_FILE = 'config_dir_redirect';

config.NSFS_NC_DEFAULT_CONF_DIR = '/etc/noobaa.conf.d';

config.NSFS_NC_CONF_DIR = process.env.NSFS_NC_CONF_DIR || '';

config.NSFS_TEMP_CONF_DIR_NAME = '.noobaa-config-nsfs';

config.NSFS_NC_CONFIG_DIR_BACKEND = '';

config.NSFS_NC_STORAGE_BACKEND = '';

config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001;

config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;

config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;

config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1;

config.ALLOW_HTTP = false;

// config files should allow access to the owner of the files

config.BASE_MODE_CONFIG_FILE = 0o600;

config.BASE_MODE_CONFIG_DIR = 0o700;

config.S3_SERVER_IP_WHITELIST = [];

config.VIRTUAL_HOSTS = process.env.VIRTUAL_HOSTS || '';

config.NC_HEALTH_ENDPOINT_RETRY_COUNT = 3;

config.NC_HEALTH_ENDPOINT_RETRY_DELAY = 10;

/** @type {'file' | 'executable'} */

config.NC_MASTER_KEYS_STORE_TYPE = 'file';

// unless override in config.json, the default will be the config_dir/master_keys.json

config.NC_MASTER_KEYS_FILE_LOCATION = '';

config.NC_MASTER_KEYS_GET_EXECUTABLE = '';

config.NC_MASTER_KEYS_PUT_EXECUTABLE = '';

config.NC_MASTER_KEYS_MANAGER_REFRESH_THRESHOLD = -1; // currently we want to disable automatic refresh

config.MASTER_KEYS_EXEC_MAX_RETRIES = 3;

config.NC_DISABLE_ACCESS_CHECK = false;

config.NC_DISABLE_HEALTH_ACCESS_CHECK = false;

config.NC_DISABLE_POSIX_MODE_ACCESS_CHECK = true;

config.NC_DISABLE_SCHEMA_CHECK = false;

config.ACCOUNTS_ID_CACHE_EXPIRY = 3 * 60 * 1000;

////////// GPFS //////////

config.GPFS_DOWN_DELAY = 1000;

//Quota

config.QUOTA_LOW_THRESHOLD = 80;

config.QUOTA_MAX_OBJECTS = Number.MAX_SAFE_INTEGER;

//////////////////////////

// STS CONFIG //

//////////////////////////

config.STS_DEFAULT_SESSION_TOKEN_EXPIRY_MS = 60 * 60 * 1000; // 1 hour

config.STS_MIN_DURATION_SECONDS = 15 * 60; // 15 minutes

config.STS_MAX_DURATION_SECONDS = 12 * 60 * 60; // 12 hours

/////////////////////////

// BLOCK STORE FS //

/////////////////////////

config.BLOCK_STORE_CACHE_SIZE = 256 * 1024 * 1024;

config.BLOCK_STORE_FS_CACHED_DF_MAX_TIME = 30 * 1000; // 30 seconds

config.BLOCK_STORE_FS_CACHED_DF_MIN_TIME = 1 * 1000; // 1 seconds

config.BLOCK_STORE_FS_CACHED_DF_MIN_SPACE = 1 * 1024 * 1024 * 1024; // 1 GB

config.BLOCK_STORE_FS_TMFS_ENABLED = false;

config.BLOCK_STORE_FS_MAPPING_INFO_ENABLED = false;

config.BLOCK_STORE_FS_TMFS_ALLOW_MIGRATED_READS = true;

config.BLOCK_STORE_FS_XATTR_BLOCK_MD = 'user.noobaa.block_md';

config.BLOCK_STORE_FS_XATTR_QUERY_MIGSTAT = 'user._query.migstat';

config.BLOCK_STORE_FS_XATTR_TRIGGER_RECALL = 'user._trigger.recall';

config.BLOCK_STORE_FS_XATTR_TRIGGER_MIGRATE = 'user._trigger.migrate';

config.BLOCK_STORE_FS_XATTR_TRIGGER_PREMIGRATE = 'user._trigger.premigrate';

config.TIERING_TTL_WORKER_ENABLED = false;

config.TIERING_TTL_WORKER_BATCH_SIZE = 1000;

config.TIERING_TTL_WORKER_BATCH_DELAY = 1 * 60 * 1000; // 1 minutes

config.TIERING_TTL_MS = 30 * 60 * 1000; // 30 minutes

/////////////////////////

// AWS SDK VERSION //

/////////////////////////

config.AWS_SDK_VERSION_3_ENABLED = true;

config.DEFAULT_REGION = 'us-east-1';

/////////////////////////

/// VACCUM ANALYZER ///

/////////////////////////

config.VACCUM_ANALYZER_INTERVAL = 86400000;

/////////////////////

// //

// OVERRIDES //

// //

// KEEP ME LAST! //

// //

/////////////////////

// load a local config file that overwrites some of the config

function load_config_local() {