diff --git a/cluster_connect b/cluster_connect index 2e5dbc1ec1..e361072da5 100755 --- a/cluster_connect +++ b/cluster_connect @@ -35,6 +35,8 @@ def usage(): -I default: 256\n\ -t (ephemeral, membase, memcached)\n\ default: membase\n\ + -S (couchstore, magma)\n\ + default: couchstore\n\ -r (max 3) default: 1\n\ (Only for ephemeral or membase buckets!)\n\ -i (don't index replicas) default: replica index enabled\n\ @@ -52,7 +54,7 @@ class PasswordManager(urllib2.HTTPPasswordMgr): def main(): try: opts, args = getopt.getopt(sys.argv[1:], - "n:t:s:r:iT:I:p:", + "n:t:s:r:iT:S:I:p:", ["dont-rebalance", "afamily=", "enable-encryption"]) except getopt.GetoptError as err: @@ -61,6 +63,7 @@ def main(): sys.exit() nodes = 0 buckettype = "membase" + storage_backend = "couchstore" memsize = 256 indexmemsize = 256 replicas = 1 @@ -99,6 +102,10 @@ def main(): encryption = True elif o == "--dont-rebalance": do_rebalance = False + elif o == "-S": + if a not in ("couchstore", "magma"): + assert False, "invalid storage backend" + storage_backend = a else: usage() sys.exit() @@ -155,6 +162,7 @@ def main(): "&authType=sasl" + "&saslPassword=" + "&bucketType=" + buckettype + + "&storageBackend=" + storage_backend + "&ramQuotaMB=" + str(memsize) + "&replicaNumber=" + str(replicas) + "&replicaIndex=" + bool_request_value(replica_index)).read() diff --git a/priv/public/ui/app-new/mn-buckets-dialog.html b/priv/public/ui/app-new/mn-buckets-dialog.html index 3e64e8cc46..da161c7502 100644 --- a/priv/public/ui/app-new/mn-buckets-dialog.html +++ b/priv/public/ui/app-new/mn-buckets-dialog.html @@ -487,6 +487,44 @@

Edit Bucket Settings

+
+ + + The type of storage backend to use for this bucket. + + + + + +
+ + + + +
+ {{(storageBackendWarning | async)}} +
+
+
+ +
Cancel diff --git a/priv/public/ui/app/components/directives/mn_buckets_form/mn_buckets_form.html b/priv/public/ui/app/components/directives/mn_buckets_form/mn_buckets_form.html index ebc55e2f0b..100634f3c7 100644 --- a/priv/public/ui/app/components/directives/mn_buckets_form/mn_buckets_form.html +++ b/priv/public/ui/app/components/directives/mn_buckets_form/mn_buckets_form.html @@ -453,5 +453,42 @@ ng-true-value="1">
+ +
+ + + + + +
+ + + + +
+ {{validation.result.errors.storageBackend}} +
+
+
diff --git a/priv/public/ui/app/constants/constants.js b/priv/public/ui/app/constants/constants.js index a6409463f2..d0a78971c4 100644 --- a/priv/public/ui/app/constants/constants.js +++ b/priv/public/ui/app/constants/constants.js @@ -19,7 +19,8 @@ uri: '/pools/default/buckets', purgeInterval: "3", compressionMode: "passive", - maxTTL: 0 + maxTTL: 0, + storageBackend: 'couchstore', }) .constant('daysOfWeek', [ 'Monday', diff --git a/priv/public/ui/app/mn_admin/mn_buckets/details/mn_buckets_details.html b/priv/public/ui/app/mn_admin/mn_buckets/details/mn_buckets_details.html index 484cd4a8da..86049655aa 100644 --- a/priv/public/ui/app/mn_admin/mn_buckets/details/mn_buckets_details.html +++ b/priv/public/ui/app/mn_admin/mn_buckets/details/mn_buckets_details.html @@ -50,6 +50,13 @@ Metadata Purge Interval: {{(bucketsDetailsCtl.bucketDetails.purgeInterval)}} +
+ Storage Backend: + {{bucketsDetailsCtl.bucketDetails.storageBackend === "couchstore" ? "CouchStore" : + bucketsDetailsCtl.bucketDetails.storageBackend === "magma" ? "Magma" : ""}} +
diff --git a/priv/public/ui/app/mn_admin/mn_buckets/details_dialog/mn_buckets_details_dialog_service.js b/priv/public/ui/app/mn_admin/mn_buckets/details_dialog/mn_buckets_details_dialog_service.js index a08750a134..c9e76a8e59 100644 --- a/priv/public/ui/app/mn_admin/mn_buckets/details_dialog/mn_buckets_details_dialog_service.js +++ b/priv/public/ui/app/mn_admin/mn_buckets/details_dialog/mn_buckets_details_dialog_service.js @@ -41,6 +41,7 @@ } if (bucketConf.bucketType === "membase") { copyProperties(["autoCompactionDefined", "evictionPolicy"]); + copyProperty("storageBackend"); } if (bucketConf.bucketType === "ephemeral") { copyProperty("purgeInterval"); diff --git a/src/bucket_info_cache.erl b/src/bucket_info_cache.erl index 36002a4f82..f42beda5f3 100644 --- a/src/bucket_info_cache.erl +++ b/src/bucket_info_cache.erl @@ -201,11 +201,11 @@ compute_bucket_info_with_config(Bucket, Config, BucketConfig) -> {struct, VBMap} = ns_bucket:json_map_with_full_config(?LOCALHOST_MARKER_STRING, BucketConfig, Config), VBMapInfo = [{vBucketServerMap, {VBMap}} | Caps], - case ns_bucket:storage_mode(BucketConfig) of - couchstore -> - [{ddocs, {[{uri, <<"/pools/default/buckets/", BucketBin/binary, "/ddocs">>}]}} - | VBMapInfo]; - _ -> + case ns_bucket:can_have_views(BucketConfig) of + true -> + [{ddocs, {[{uri, <<"/pools/default/buckets/", BucketBin/binary, + "/ddocs">>}]}} | VBMapInfo]; + false -> VBMapInfo end end, diff --git a/src/diag_handler.erl b/src/diag_handler.erl index eba2f39623..439230426e 100644 --- a/src/diag_handler.erl +++ b/src/diag_handler.erl @@ -291,7 +291,8 @@ collect_diag_per_node_loop(ReplyRef, ChildRef, Results) -> collect_diag_per_node_body(Reply) -> ?log_debug("Start collecting diagnostic data"), ActiveBuckets = ns_memcached:active_buckets(), - PersistentBuckets = [B || B <- ActiveBuckets, ns_bucket:is_persistent(B)], + PersistentBuckets = [B || B <- ActiveBuckets, + ns_bucket:is_named_bucket_persistent(B)], Reply(processes, grab_process_infos()), Reply(babysitter_processes, (catch grab_babysitter_process_infos())), diff --git a/src/memcached_bucket_config.erl b/src/memcached_bucket_config.erl index 19f06a92d6..311a39c098 100644 --- a/src/memcached_bucket_config.erl +++ b/src/memcached_bucket_config.erl @@ -30,8 +30,6 @@ get_current_collections_uid/1]). params(membase, BucketName, BucketConfig, MemQuota, UUID) -> - StorageMode = ns_bucket:storage_mode(BucketConfig), - {DriftAheadThreshold, DriftBehindThreshold} = case ns_bucket:drift_thresholds(BucketConfig) of undefined -> @@ -44,7 +42,7 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) -> [{"max_size", [{reload, flush}], MemQuota}, {"dbname", [restart], DBSubDir}, - {"backend", [], couchdb}, + {"backend", [], ns_bucket:kv_backend_type(BucketConfig)}, {"couch_bucket", [], BucketName}, {"max_vbuckets", [], proplists:get_value(num_vbuckets, BucketConfig)}, {"alog_path", [], filename:join(DBSubDir, "access.log")}, @@ -54,17 +52,17 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) -> {"uuid", [], UUID}, {"conflict_resolution_type", [], ns_bucket:conflict_resolution_type(BucketConfig)}, - {"bucket_type", [], storage_mode_to_bucket_type(StorageMode)}, + {"bucket_type", [], ns_bucket:kv_bucket_type(BucketConfig)}, {"hlc_drift_ahead_threshold_us", [no_param, {reload, vbucket}], DriftAheadThreshold}, {"hlc_drift_behind_threshold_us", [no_param, {reload, vbucket}], DriftBehindThreshold}, {"item_eviction_policy", maybe_restart(), - item_eviction_policy(StorageMode, BucketConfig)}, + get_eviction_policy(true, BucketConfig)}, {"ephemeral_full_policy", [{reload, flush}], - ephemeral_full_policy(StorageMode, BucketConfig)}, + get_eviction_policy(false, BucketConfig)}, {"ephemeral_metadata_purge_age", [{reload, flush}], - ephemeral_metadata_purge_age(StorageMode, BucketConfig)}, + ephemeral_metadata_purge_age(BucketConfig)}, {"max_ttl", [{reload, flush}], proplists:get_value(max_ttl, BucketConfig)}, {"ht_locks", [], proplists:get_value( ht_locks, BucketConfig, @@ -74,15 +72,11 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) -> misc:getenv_int("MEMBASE_HT_SIZE", undefined))}, {"compression_mode", [{reload, flush}], proplists:get_value(compression_mode, BucketConfig)}]; + params(memcached, _BucketName, _BucketConfig, MemQuota, UUID) -> [{"cache_size", [], MemQuota}, {"uuid", [], UUID}]. -storage_mode_to_bucket_type(couchstore) -> - persistent; -storage_mode_to_bucket_type(ephemeral) -> - ephemeral. - maybe_restart() -> case ns_config:read_key_fast(dont_reload_bucket_on_cfg_change, false) of false -> @@ -91,29 +85,32 @@ maybe_restart() -> [] end. -item_eviction_policy(couchstore, BucketConfig) -> - proplists:get_value(eviction_policy, BucketConfig, value_only); -item_eviction_policy(ephemeral, _BucketConfig) -> - undefined. - -ephemeral_full_policy(ephemeral, BucketConfig) -> - case proplists:get_value(eviction_policy, BucketConfig, no_eviction) of - nru_eviction -> - auto_delete; - no_eviction -> - fail_new_data - end; -ephemeral_full_policy(couchstore, _BucketConfig) -> - undefined. +get_eviction_policy(Persistent, BucketConfig) -> + case ns_bucket:is_persistent(BucketConfig) of + Persistent -> + case ns_bucket:eviction_policy(BucketConfig) of + nru_eviction -> + auto_delete; + no_eviction -> + fail_new_data; + Other -> + Other + end; + _ -> + undefined + end. -ephemeral_metadata_purge_age(ephemeral, BucketConfig) -> - %% Purge interval is accepted in # of days but the ep-engine - %% needs it to be expressed in seconds. - Val = proplists:get_value(purge_interval, BucketConfig, - ?DEFAULT_EPHEMERAL_PURGE_INTERVAL_DAYS), - erlang:round(Val * 24 * 3600); -ephemeral_metadata_purge_age(couchstore, _BucketConfig) -> - undefined. +ephemeral_metadata_purge_age(BucketConfig) -> + case ns_bucket:is_persistent(BucketConfig) of + true -> + undefined; + false -> + %% Purge interval is accepted in # of days but the ep-engine + %% needs it to be expressed in seconds. + Val = proplists:get_value(purge_interval, BucketConfig, + ?DEFAULT_EPHEMERAL_PURGE_INTERVAL_DAYS), + erlang:round(Val * 24 * 3600) + end. get(Config, BucketName) -> {ok, BucketConfig} = ns_bucket:get_bucket(BucketName, Config), diff --git a/src/menelaus_stats.erl b/src/menelaus_stats.erl index 0972ffefa8..68d02a4c9a 100644 --- a/src/menelaus_stats.erl +++ b/src/menelaus_stats.erl @@ -1,5 +1,5 @@ %% @author Couchbase -%% @copyright 2009-2018 Couchbase, Inc. +%% @copyright 2009-2020 Couchbase, Inc. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -622,7 +622,7 @@ section_nodes(Section) -> is_persistent("@"++_) -> false; is_persistent(BucketName) -> - ns_bucket:is_persistent(BucketName). + ns_bucket:is_named_bucket_persistent(BucketName). bucket_exists(Bucket) -> ns_bucket:get_bucket(Bucket) =/= not_present. @@ -2467,7 +2467,7 @@ server_resources_stats_description(ServiceNodes) -> base_stats_directory(BucketId, ServiceNodes) -> {ok, BucketConfig} = ns_bucket:get_bucket(BucketId), - Base = case menelaus_web_buckets:external_bucket_type(BucketConfig) of + Base = case ns_bucket:external_bucket_type(BucketConfig) of membase -> membase_stats_description(BucketId, ServiceNodes); memcached -> memcached_stats_description(); ephemeral -> ephemeral_stats_description(BucketId, ServiceNodes) diff --git a/src/menelaus_web_buckets.erl b/src/menelaus_web_buckets.erl index 87f154b338..b29f8b70ec 100644 --- a/src/menelaus_web_buckets.erl +++ b/src/menelaus_web_buckets.erl @@ -1,5 +1,5 @@ %% @author Couchbase -%% @copyright 2009-2019 Couchbase, Inc. +%% @copyright 2009-2020 Couchbase, Inc. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -49,8 +49,6 @@ handle_set_ddoc_update_min_changes/4, handle_local_random_key/3, build_bucket_capabilities/1, - external_bucket_type/1, - display_type/2, maybe_cleanup_old_buckets/0, serve_short_bucket_info/2, serve_streaming_short_bucket_info/2, @@ -176,59 +174,59 @@ add_couch_api_base(BucketName, BucketUUID, KV, Node, LocalAddr) -> end end, KV, NodesKeysList). -%% Used while building the bucket info. This transforms the internal -%% representation of bucket types to externally known bucket types. -%% Ideally the 'display_type' function should suffice here but there -%% is too much reliance on the atom membase by other modules (ex: xdcr). -external_bucket_type(BucketConfig) -> - external_bucket_type(ns_bucket:bucket_type(BucketConfig), BucketConfig). - -external_bucket_type(memcached = _Type, _) -> - memcached; -external_bucket_type(membase = _Type, BucketConfig) -> - case ns_bucket:storage_mode(BucketConfig) of - couchstore -> - membase; - ephemeral -> - ephemeral - end. - -build_auto_compaction_info(BucketConfig, couchstore) -> - ACSettings = case proplists:get_value(autocompaction, BucketConfig) of - undefined -> false; - false -> false; - ACSettingsX -> ACSettingsX - end, +build_auto_compaction_info(BucketConfig) -> + case ns_bucket:is_persistent(BucketConfig) of + true -> + ACSettings = case proplists:get_value(autocompaction, + BucketConfig) of + undefined -> false; + false -> false; + ACSettingsX -> ACSettingsX + end, - case ACSettings of + case ACSettings of + false -> + [{autoCompactionSettings, false}]; + _ -> + [{autoCompactionSettings, + menelaus_web_autocompaction:build_bucket_settings( + ACSettings)}] + end; false -> - [{autoCompactionSettings, false}]; - _ -> - [{autoCompactionSettings, - menelaus_web_autocompaction:build_bucket_settings(ACSettings)}] - end; -build_auto_compaction_info(_BucketConfig, ephemeral) -> - []; -build_auto_compaction_info(_BucketConfig, undefined) -> - %% When the bucket type is memcached. - [{autoCompactionSettings, false}]. + case ns_bucket:storage_mode(BucketConfig) of + ephemeral -> + []; + undefined -> + %% When the bucket type is memcached. + [{autoCompactionSettings, false}] + end + end. -build_purge_interval_info(BucketConfig, couchstore) -> - case proplists:get_value(autocompaction, BucketConfig, false) of +build_purge_interval_info(BucketConfig) -> + case ns_bucket:is_persistent(BucketConfig) of + true -> + case proplists:get_value(autocompaction, BucketConfig, false) of + false -> + []; + _Val -> + PInterval = case proplists:get_value(purge_interval, + BucketConfig) of + undefined -> + compaction_api:get_purge_interval(global); + PI -> PI + end, + [{purgeInterval, PInterval}] + end; false -> - []; - _Val -> - PInterval = case proplists:get_value(purge_interval, BucketConfig) of - undefined -> compaction_api:get_purge_interval(global); - PI -> PI - end, - [{purgeInterval, PInterval}] - end; -build_purge_interval_info(BucketConfig, ephemeral) -> - [{purgeInterval, proplists:get_value(purge_interval, BucketConfig)}]; -build_purge_interval_info(_BucketConfig, undefined) -> - %% When the bucket type is memcached. - []. + case ns_bucket:storage_mode(BucketConfig) of + ephemeral -> + [{purgeInterval, proplists:get_value(purge_interval, + BucketConfig)}]; + undefined -> + %% When the bucket type is memcached. + [] + end + end. build_eviction_policy(BucketConfig) -> case ns_bucket:eviction_policy(BucketConfig) of @@ -287,6 +285,7 @@ build_bucket_info(Id, BucketConfig, InfoLevel, LocalAddr, MayExposeAuth, {rawRAM, ns_bucket:raw_ram_quota(BucketConfig)}]}}, {basicStats, {struct, BasicStats}}, {evictionPolicy, EvictionPolicy}, + {storageBackend, ns_bucket:storage_backend(BucketConfig)}, {conflictResolutionType, ConflictResolutionType} | BucketCaps], @@ -330,22 +329,12 @@ build_bucket_info(Id, BucketConfig, InfoLevel, LocalAddr, MayExposeAuth, memcached -> Suffix end, + ACInfo = build_auto_compaction_info(BucketConfig), + PIInfo = build_purge_interval_info(BucketConfig), + CanHaveViews = ns_bucket:can_have_views(BucketConfig), - StorageMode = ns_bucket:storage_mode(BucketConfig), - ACInfo = build_auto_compaction_info(BucketConfig, StorageMode), - PIInfo = build_purge_interval_info(BucketConfig, StorageMode), - Suffix2 = ACInfo ++ PIInfo ++ Suffix1 , - - Suffix3 = case StorageMode of - couchstore -> - DDocsURI = bin_concat_path(["pools", "default", "buckets", - Id, "ddocs"]), - [{ddocs, {struct, [{uri, DDocsURI}]}}, - {replicaIndex, proplists:get_value(replica_index, BucketConfig, true)} - | Suffix2]; - _ -> - Suffix2 - end, + Suffix2 = build_ddocs_uri(CanHaveViews, Id, + BucketConfig) ++ ACInfo ++ PIInfo ++ Suffix1, FlushEnabled = proplists:get_value(flush_enabled, BucketConfig, false), MaybeFlushController = @@ -357,18 +346,18 @@ build_bucket_info(Id, BucketConfig, InfoLevel, LocalAddr, MayExposeAuth, [] end, - Suffix4 = case MayExposeAuth of + Suffix3 = case MayExposeAuth of true -> [{saslPassword, list_to_binary(proplists:get_value(sasl_password, BucketConfig, ""))} | - Suffix3]; + Suffix2]; false -> - Suffix3 + Suffix2 end, {struct, [{name, list_to_binary(Id)}, {uuid, BucketUUID}, - {bucketType, external_bucket_type(BucketType, BucketConfig)}, + {bucketType, ns_bucket:external_bucket_type(BucketConfig)}, {authType, misc:expect_prop_value(auth_type, BucketConfig)}, {uri, BuildUUIDURI(["pools", "default", "buckets", Id])}, {streamingUri, BuildUUIDURI(["pools", "default", "bucketsStreaming", Id])}, @@ -390,7 +379,15 @@ build_bucket_info(Id, BucketConfig, InfoLevel, LocalAddr, MayExposeAuth, {directoryURI, StatsDirectoryUri}, {nodeStatsListURI, NodeStatsListURI}]}}, {nodeLocator, ns_bucket:node_locator(BucketConfig)} - | Suffix4]}. + | Suffix3]}. + +build_ddocs_uri(true, Id, BucketConfig) -> + DDocsURI = bin_concat_path(["pools", "default", "buckets", + Id, "ddocs"]), + [{ddocs, {struct, [{uri, DDocsURI}]}}, + {replicaIndex, proplists:get_value(replica_index, BucketConfig, true)}]; +build_ddocs_uri(false, _Id, _BucketConfig) -> + []. build_bucket_capabilities(BucketConfig) -> Caps = @@ -399,8 +396,7 @@ build_bucket_capabilities(BucketConfig) -> Conditional = [{collections, collections:enabled(BucketConfig)}, {durableWrite, cluster_compat_mode:is_cluster_65()}, - {couchapi, - ns_bucket:storage_mode(BucketConfig) =:= couchstore}], + {couchapi, ns_bucket:can_have_views(BucketConfig)}], [C || {C, true} <- Conditional] ++ [dcp, cbhello, touch, cccp, xdcrCheckpointing, nodesExt, @@ -590,7 +586,8 @@ handle_bucket_update_inner(BucketId, Req, Params, Limit) -> BucketId, UpdatedProps) of ok -> ns_audit:modify_bucket(Req, BucketId, BucketType, UpdatedProps), - DisplayBucketType = display_type(BucketType, StorageMode), + DisplayBucketType = ns_bucket:display_type(BucketType, + StorageMode), ale:info(?USER_LOGGER, "Updated bucket \"~s\" (of type ~s) properties:~n~p", [BucketId, DisplayBucketType, lists:keydelete(sasl_password, 1, UpdatedProps)]), @@ -641,7 +638,7 @@ do_bucket_create(Req, Name, ParsedProps) -> case ns_orchestrator:create_bucket(BucketType, Name, BucketProps) of ok -> ns_audit:create_bucket(Req, Name, BucketType, BucketProps), - DisplayBucketType = display_type(BucketType, StorageMode), + DisplayBucketType = ns_bucket:display_type(BucketType, StorageMode), ?MENELAUS_WEB_LOG(?BUCKET_CREATED, "Created bucket \"~s\" of type: ~s~n~p", [Name, DisplayBucketType, lists:keydelete(sasl_password, 1, BucketProps)]), ok; @@ -746,20 +743,6 @@ num_replicas_warnings_validation(Ctx, NReplicas) -> [{replicaNumber, ?l2b("Warning: " ++ Msg ++ ".")}] end. -%% Default bucket type is now couchbase and not membase. Ideally, we should -%% change the default bucket type atom to couchbase but the bucket type membase -%% is used/checked at multiple locations. For similar reasons, the ephemeral -%% bucket type also gets stored as 'membase' and to differentiate between the -%% couchbase and ephemeral buckets we store an extra parameter called -%% 'storage_mode'. So to fix the log message to display the correct bucket type -%% we use both type and storage_mode parameters of the bucket config. -display_type(membase = _Type, couchstore = _StorageMode) -> - couchbase; -display_type(membase = _Type, ephemeral = _StorageMode) -> - ephemeral; -display_type(Type, _) -> - Type. - handle_bucket_flush(_PoolId, Id, Req) -> XDCRDocs = goxdcr_rest:find_all_replication_docs(), case lists:any( @@ -1041,7 +1024,7 @@ validate_bucket_purge_interval(Params, _BucketConfig, true = IsNew) -> BucketType = proplists:get_value("bucketType", Params, "membase"), parse_validate_bucket_purge_interval(Params, BucketType, IsNew); validate_bucket_purge_interval(Params, BucketConfig, false = IsNew) -> - BucketType = external_bucket_type(BucketConfig), + BucketType = ns_bucket:external_bucket_type(BucketConfig), parse_validate_bucket_purge_interval(Params, atom_to_list(BucketType), IsNew). parse_validate_bucket_purge_interval(Params, "couchbase", IsNew) -> @@ -1122,15 +1105,26 @@ validate_replicas_number(Params, IsNew) -> get_storage_mode(Params, _BucketConfig, true = _IsNew) -> case proplists:get_value("bucketType", Params, "membase") of "membase" -> - {ok, storage_mode, couchstore}; + get_storage_mode_based_on_storage_backend(Params); "couchbase" -> - {ok, storage_mode, couchstore}; + get_storage_mode_based_on_storage_backend(Params); "ephemeral" -> {ok, storage_mode, ephemeral} end; get_storage_mode(_Params, BucketConfig, false = _IsNew)-> {ok, storage_mode, ns_bucket:storage_mode(BucketConfig)}. +get_storage_mode_based_on_storage_backend(Params) -> + case proplists:get_value("storageBackend", Params, "couchstore") of + "couchstore" -> + {ok, storage_mode, couchstore}; + "magma" -> + {ok, storage_mode, magma}; + _ -> + {error, storage_mode, + <<"storage backend must be couchstore or magma">>} + end. + get_conflict_resolution_type_and_thresholds(Params, _BucketConfig, true = IsNew) -> case proplists:get_value("conflictResolutionType", Params) of undefined -> @@ -1407,7 +1401,7 @@ parse_validate_threads_number(NumThreads) -> parse_validate_eviction_policy(Params, BCfg, IsNew) -> BType = case IsNew of true -> proplists:get_value("bucketType", Params, "membase"); - false -> atom_to_list(external_bucket_type(BCfg)) + false -> atom_to_list(ns_bucket:external_bucket_type(BCfg)) end, do_parse_validate_eviction_policy(Params, BCfg, BType, IsNew). diff --git a/src/menelaus_web_pools.erl b/src/menelaus_web_pools.erl index d958ef0ef5..2890b0f411 100644 --- a/src/menelaus_web_pools.erl +++ b/src/menelaus_web_pools.erl @@ -1,5 +1,5 @@ %% @author Couchbase -%% @copyright 2017-2018 Couchbase, Inc. +%% @copyright 2017-2020 Couchbase, Inc. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -557,11 +557,7 @@ extract_bucket_specific_data(BucketCfgs, ExtractFun) -> undefined -> Acc; Val -> - StorageMode = ns_bucket:storage_mode(BCfg), - BucketType = ns_bucket:bucket_type(BCfg), - DisplayType = - menelaus_web_buckets:display_type(BucketType, - StorageMode), + DisplayType = ns_bucket:display_type(BCfg), case proplists:get_value(DisplayType, Acc) of undefined -> diff --git a/src/ns_bucket.erl b/src/ns_bucket.erl index 94d3f49f20..e0b5212121 100644 --- a/src/ns_bucket.erl +++ b/src/ns_bucket.erl @@ -1,5 +1,5 @@ %% @author Couchbase -%% @copyright 2009-2019 Couchbase, Inc. +%% @copyright 2009-2020 Couchbase, Inc. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -27,10 +27,15 @@ -export([auth_type/1, get_servers/1, bucket_type/1, + kv_bucket_type/1, + kv_backend_type/1, num_replicas_changed/1, create_bucket/3, credentials/1, delete_bucket/1, + display_type/1, + display_type/2, + external_bucket_type/1, failover_warnings/0, get_bucket/1, get_bucket/2, @@ -41,6 +46,7 @@ get_bucket_names_of_type/2, get_buckets/0, get_buckets/1, + is_named_bucket_persistent/1, is_persistent/1, is_valid_bucket_name/1, json_map_from_config/2, @@ -61,6 +67,7 @@ drift_thresholds/1, eviction_policy/1, storage_mode/1, + storage_backend/1, raw_ram_quota/1, sasl_password/1, set_bucket_config/2, @@ -133,6 +140,7 @@ get_bucket_names(BucketConfigs) -> proplists:get_keys(BucketConfigs). -type bucket_type_mode() :: memcached|membase|{membase, couchstore}| + {membase, magma}| {membase, ephemeral}| {memcached, undefined}. -spec get_bucket_names_of_type(bucket_type_mode()) -> list(). @@ -178,6 +186,7 @@ eviction_policy(BucketConfig) -> Default = case storage_mode(BucketConfig) of undefined -> value_only; couchstore -> value_only; + magma -> value_only; ephemeral -> no_eviction end, proplists:get_value(eviction_policy, BucketConfig, Default). @@ -191,6 +200,22 @@ storage_mode(BucketConfig) -> proplists:get_value(storage_mode, BucketConfig, couchstore) end. +-spec storage_backend([{_,_}]) -> atom(). +storage_backend(BucketConfig) -> + BucketType = bucket_type(BucketConfig), + StorageMode = storage_mode(BucketConfig), + case BucketType of + membase -> + case StorageMode of + ephemeral -> + undefined; + SM -> + SM + end; + memcached -> + undefined + end. + %% returns bucket ram quota multiplied by number of nodes this bucket %% resides on. I.e. gives amount of ram quota that will be used by %% across the cluster for this bucket. @@ -361,9 +386,65 @@ num_replicas(Bucket) -> X end. +%% ns_server type (membase vs memcached) bucket_type(Bucket) -> proplists:get_value(type, Bucket). +%% KV type (persistent vs ephemeral) +kv_bucket_type(BucketConfig) -> + case is_persistent(BucketConfig) of + true -> persistent; + false -> ephemeral + end. + +kv_backend_type(BucketConfig) -> + StorageMode = storage_mode(BucketConfig), + case StorageMode of + couchstore -> couchdb; + magma -> magma; + %% KV requires a value but only accepts: couchdb, magma, rocksdb. + %% So we've always passed couchdb for ephemeral buckets which KV + %% will parse as an acceptable value but not use it. + ephemeral -> couchdb + end. + +%% Used for REST API compatibility. This transforms the internal +%% representation of bucket types to externally known bucket types. +%% Ideally the 'display_type' function should suffice here but there +%% is too much reliance on the atom membase by other modules (ex: xdcr). +external_bucket_type(BucketConfig) -> + BucketType = bucket_type(BucketConfig), + case BucketType of + memcached -> memcached; + membase -> + case storage_mode(BucketConfig) of + couchstore -> membase; + magma -> membase; + ephemeral -> ephemeral + end + end. + +%% Default bucket type is now couchbase and not membase. Ideally, we should +%% change the default bucket type atom to couchbase but the bucket type membase +%% is used/checked at multiple locations. For similar reasons, the ephemeral +%% bucket type also gets stored as 'membase' and to differentiate between the +%% couchbase and ephemeral buckets we store an extra parameter called +%% 'storage_mode'. So to fix the log message to display the correct bucket type +%% we use both type and storage_mode parameters of the bucket config. +display_type(BucketConfig) -> + BucketType = bucket_type(BucketConfig), + StorageMode = storage_mode(BucketConfig), + display_type(BucketType, StorageMode). + +display_type(membase = _Type, couchstore = _StorageMode) -> + couchbase; +display_type(membase = _Type, magma = _StorageMode) -> + couchbase; +display_type(membase = _Type, ephemeral = _StorageMode) -> + ephemeral; +display_type(Type, _) -> + Type. + auth_type(Bucket) -> proplists:get_value(auth_type, Bucket). @@ -650,10 +731,14 @@ update_bucket_config(BucketName, Fun) -> RV end). -is_persistent(BucketName) -> +is_named_bucket_persistent(BucketName) -> {ok, BucketConfig} = get_bucket(BucketName), + is_persistent(BucketConfig). + +is_persistent(BucketConfig) -> bucket_type(BucketConfig) =:= membase andalso - storage_mode(BucketConfig) =:= couchstore. + (storage_mode(BucketConfig) =:= couchstore orelse + storage_mode(BucketConfig) =:= magma). names_conflict(BucketNameA, BucketNameB) -> string:to_lower(BucketNameA) =:= string:to_lower(BucketNameB). @@ -676,12 +761,13 @@ node_bucket_names(Node) -> node_bucket_names(Node, get_buckets()). -spec node_bucket_names_of_type(node(), memcached|membase, - undefined|couchstore|ephemeral) -> list(). + undefined|couchstore|magma|ephemeral) -> list(). node_bucket_names_of_type(Node, Type, Mode) -> node_bucket_names_of_type(Node, Type, Mode, get_buckets()). -spec node_bucket_names_of_type(node(), memcached|membase, - undefined|couchstore|ephemeral, list()) -> list(). + undefined|couchstore|magma|ephemeral, + list()) -> list(). node_bucket_names_of_type(Node, Type, Mode, BucketConfigs) -> [B || {B, C} <- BucketConfigs, lists:member(Node, get_servers(C)), @@ -774,7 +860,8 @@ is_compatible_past_map(Nodes, BucketConfig, Map) -> lists:member(Map, Matching). can_have_views(BucketConfig) -> - storage_mode(BucketConfig) =:= couchstore. + storage_mode(BucketConfig) =:= couchstore orelse + storage_mode(BucketConfig) =:= magma. bucket_view_nodes(Bucket) -> bucket_view_nodes(Bucket, ns_config:latest()).