Magma

cluster_connect

@@ -35,6 +35,8 @@ def usage():
             -I <index memory size> default: 256\n\
             -t <bucket type> (ephemeral, membase, memcached)\n\
                default: membase\n\
+            -S <storage backend> (couchstore, magma)\n\
+               default: couchstore\n\
             -r <num replicas> (max 3) default: 1\n\
                (Only for ephemeral or membase buckets!)\n\
             -i (don't index replicas) default: replica index enabled\n\
@@ -52,7 +54,7 @@ class PasswordManager(urllib2.HTTPPasswordMgr):
 def main():
     try:
         opts, args = getopt.getopt(sys.argv[1:],
-                                   "n:t:s:r:iT:I:p:",
+                                   "n:t:s:r:iT:S:I:p:",
                                    ["dont-rebalance", "afamily=",
                                     "enable-encryption"])
     except getopt.GetoptError as err:
@@ -61,6 +63,7 @@ def main():
         sys.exit()
     nodes = 0
     buckettype = "membase"
+    storage_backend = "couchstore"
     memsize = 256
     indexmemsize = 256
     replicas = 1
@@ -99,6 +102,10 @@ def main():
             encryption = True
         elif o == "--dont-rebalance":
             do_rebalance = False
+        elif o == "-S":
+            if a not in ("couchstore", "magma"):
+                assert False, "invalid storage backend"
+            storage_backend = a
         else:
             usage()
             sys.exit()
@@ -155,6 +162,7 @@ def main():
            "&authType=sasl" +
            "&saslPassword=" +
            "&bucketType=" + buckettype +
+           "&storageBackend=" + storage_backend +
            "&ramQuotaMB=" + str(memsize) +
            "&replicaNumber=" + str(replicas) +
            "&replicaIndex=" + bool_request_value(replica_index)).read()

priv/public/ui/app-new/mn-buckets-dialog.html

@@ -487,6 +487,44 @@ <h2 [hidden]="isNew">Edit Bucket Settings</h2>
 
       </div>
 
+      <div
+         class="formrow checkbox-list"
+         [hidden]="!(bucketsDialogForm | mnIsMembase)">
+         <label class="inline">Storage Backend</label>
+         <ng-template #storageBackendTooltip>
+             The type of storage backend to use for this bucket.
+         </ng-template>
+         <span
+            class="fa-stack icon-info"
+            [ngbTooltip]="storageBackendTooltip"
+            placement="top">
+           <span class="icon fa-circle-thin fa-stack-2x"></span>
+           <span class="icon fa-info fa-stack-1x"></span>
+         </span>
+         <div class="formrow">
+            <input
+               type="radio"
+               name="storageBackend"
+               formControlName="storageBackend"
+               value="couchstore"
+               id="for-storage-backend-couchstore">
+            <label for="for-storage-backend-couchstore">couchstore</label>
+            <input
+               type="radio"
+               name="storageBackend"
+               formControlName="storageBackend"
+               value="magma"
+               id="for-storage-backend-magma">
+            <label for="for-storage-backend-magma">magma</label>
+            <div
+               class="error"
+               [hidden]="!(storageBackendWarning | async)">
+              {{(storageBackendWarning | async)}}
+            </div>
+          </div>
+        </div>
+      </div>
+
     </div>
     <div class="panel-footer">
       <a (click)="activeModal.dismiss()">Cancel</a>

priv/public/ui/app/components/directives/mn_buckets_form/mn_buckets_form.html

@@ -453,5 +453,42 @@
          ng-true-value="1">
       <label for="for-enable-flush">Enable</label>
     </div>
+
+    <div
+       class="formrow checkbox-list"
+       ng-if="pools.isEnterprise && bucketConf.bucketType === 'membase'">
+      <label class="inline">Storage Backend</label>
+      <span
+         class="fa-stack icon-info"
+         uib-tooltip-html="'Type of storage backend'"
+         tooltip-placement="top">
+        <span class="icon fa-circle-thin fa-stack-2x"></span>
+        <span class="icon fa-info fa-stack-1x"></span>
+      </span>
+      <div class="formrow checkbox-list">
+        <input
+           type="radio"
+           name="storageBackend"
+           value="couchstore"
+           id="for-storage-backend-couchstore"
+           checked="checked"
+           ng-model="bucketConf.storageBackend"
+           ng-disabled="!bucketConf.isNew">
+        <label for="for-storage-backend-couchstore">CouchStore</label>
+        <input
+           type="radio"
+           name="storageBackend"
+           value="magma"
+           id="for-storage-backend-magma"
+           ng-model="bucketConf.storageBackend"
+           ng-disabled="!bucketConf.isNew">
+        <label for="for-storage-backend-magma">Magma</label>
+        <div
+           class="error margin-bottom-half"
+           ng-show="validation.result.errors.storageBackend">
+            {{validation.result.errors.storageBackend}}
+        </div>
+      </div>
+    </div>
   </div>
 </div>

priv/public/ui/app/constants/constants.js

@@ -19,7 +19,8 @@
       uri: '/pools/default/buckets',
       purgeInterval: "3",
       compressionMode: "passive",
-      maxTTL: 0
+      maxTTL: 0,
+      storageBackend: 'couchstore',
     })
     .constant('daysOfWeek', [
       'Monday',

priv/public/ui/app/mn_admin/mn_buckets/details/mn_buckets_details.html

@@ -50,6 +50,13 @@
         <strong>Metadata Purge Interval:</strong>
         {{(bucketsDetailsCtl.bucketDetails.purgeInterval)}}
       </div>
+      <div
+         class="nowrap"
+         ng-if="bucket.isMembase">
+        <strong>Storage Backend:</strong>
+        {{bucketsDetailsCtl.bucketDetails.storageBackend === "couchstore" ? "CouchStore" :
+          bucketsDetailsCtl.bucketDetails.storageBackend === "magma" ? "Magma" : ""}}
+      </div>
     </div>
 
     <div class="cbui-table-meters buckets">

priv/public/ui/app/mn_admin/mn_buckets/details_dialog/mn_buckets_details_dialog_service.js

@@ -41,6 +41,7 @@
       }
       if (bucketConf.bucketType === "membase") {
         copyProperties(["autoCompactionDefined", "evictionPolicy"]);
+        copyProperty("storageBackend");
       }
       if (bucketConf.bucketType === "ephemeral") {
         copyProperty("purgeInterval");

src/bucket_info_cache.erl

@@ -201,11 +201,11 @@ compute_bucket_info_with_config(Bucket, Config, BucketConfig) ->
                 {struct, VBMap} = ns_bucket:json_map_with_full_config(?LOCALHOST_MARKER_STRING,
                                                                       BucketConfig, Config),
                 VBMapInfo = [{vBucketServerMap, {VBMap}} | Caps],
-                case ns_bucket:storage_mode(BucketConfig) of
-                    couchstore ->
-                        [{ddocs, {[{uri, <<"/pools/default/buckets/", BucketBin/binary, "/ddocs">>}]}}
-                         | VBMapInfo];
-                    _ ->
+                case ns_bucket:can_have_views(BucketConfig) of
+                    true ->
+                        [{ddocs, {[{uri, <<"/pools/default/buckets/", BucketBin/binary,
+                                           "/ddocs">>}]}} | VBMapInfo];
+                    false ->
                         VBMapInfo
                 end
         end,

src/diag_handler.erl

@@ -291,7 +291,8 @@ collect_diag_per_node_loop(ReplyRef, ChildRef, Results) ->
 collect_diag_per_node_body(Reply) ->
     ?log_debug("Start collecting diagnostic data"),
     ActiveBuckets = ns_memcached:active_buckets(),
-    PersistentBuckets = [B || B <- ActiveBuckets, ns_bucket:is_persistent(B)],
+    PersistentBuckets = [B || B <- ActiveBuckets,
+                              ns_bucket:is_named_bucket_persistent(B)],
 
     Reply(processes, grab_process_infos()),
     Reply(babysitter_processes, (catch grab_babysitter_process_infos())),

src/memcached_bucket_config.erl

@@ -30,8 +30,6 @@
          get_current_collections_uid/1]).
 
 params(membase, BucketName, BucketConfig, MemQuota, UUID) ->
-    StorageMode = ns_bucket:storage_mode(BucketConfig),
-
     {DriftAheadThreshold, DriftBehindThreshold} =
         case ns_bucket:drift_thresholds(BucketConfig) of
             undefined ->
@@ -44,7 +42,7 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) ->
 
     [{"max_size", [{reload, flush}], MemQuota},
      {"dbname", [restart], DBSubDir},
-     {"backend", [], couchdb},
+     {"backend", [], ns_bucket:kv_backend_type(BucketConfig)},
      {"couch_bucket", [], BucketName},
      {"max_vbuckets", [], proplists:get_value(num_vbuckets, BucketConfig)},
      {"alog_path", [], filename:join(DBSubDir, "access.log")},
@@ -54,17 +52,17 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) ->
      {"uuid", [], UUID},
      {"conflict_resolution_type", [],
       ns_bucket:conflict_resolution_type(BucketConfig)},
-     {"bucket_type", [], storage_mode_to_bucket_type(StorageMode)},
+     {"bucket_type", [], ns_bucket:kv_bucket_type(BucketConfig)},
      {"hlc_drift_ahead_threshold_us", [no_param, {reload, vbucket}],
       DriftAheadThreshold},
      {"hlc_drift_behind_threshold_us", [no_param, {reload, vbucket}],
       DriftBehindThreshold},
      {"item_eviction_policy", maybe_restart(),
-      item_eviction_policy(StorageMode, BucketConfig)},
+      get_eviction_policy(true, BucketConfig)},
      {"ephemeral_full_policy", [{reload, flush}],
-      ephemeral_full_policy(StorageMode, BucketConfig)},
+      get_eviction_policy(false, BucketConfig)},
      {"ephemeral_metadata_purge_age", [{reload, flush}],
-      ephemeral_metadata_purge_age(StorageMode, BucketConfig)},
+      ephemeral_metadata_purge_age(BucketConfig)},
      {"max_ttl", [{reload, flush}], proplists:get_value(max_ttl, BucketConfig)},
      {"ht_locks", [], proplists:get_value(
                         ht_locks, BucketConfig,
@@ -74,15 +72,11 @@ params(membase, BucketName, BucketConfig, MemQuota, UUID) ->
                           misc:getenv_int("MEMBASE_HT_SIZE", undefined))},
      {"compression_mode", [{reload, flush}],
       proplists:get_value(compression_mode, BucketConfig)}];
+
 params(memcached, _BucketName, _BucketConfig, MemQuota, UUID) ->
     [{"cache_size", [], MemQuota},
      {"uuid", [], UUID}].
 
-storage_mode_to_bucket_type(couchstore) ->
-    persistent;
-storage_mode_to_bucket_type(ephemeral) ->
-    ephemeral.
-
 maybe_restart() ->
     case ns_config:read_key_fast(dont_reload_bucket_on_cfg_change, false) of
         false ->
@@ -91,29 +85,32 @@ maybe_restart() ->
             []
     end.
 
-item_eviction_policy(couchstore, BucketConfig) ->
-    proplists:get_value(eviction_policy, BucketConfig, value_only);
-item_eviction_policy(ephemeral, _BucketConfig) ->
-    undefined.
-
-ephemeral_full_policy(ephemeral, BucketConfig) ->
-    case proplists:get_value(eviction_policy, BucketConfig, no_eviction) of
-        nru_eviction ->
-            auto_delete;
-        no_eviction ->
-            fail_new_data
-    end;
-ephemeral_full_policy(couchstore, _BucketConfig) ->
-    undefined.
+get_eviction_policy(Persistent, BucketConfig) ->
+    case ns_bucket:is_persistent(BucketConfig) of
+        Persistent ->
+            case ns_bucket:eviction_policy(BucketConfig) of
+                nru_eviction ->
+                    auto_delete;
+                no_eviction ->
+                    fail_new_data;
+                Other ->
+                    Other
+            end;
+        _ ->
+            undefined
+    end.
 
-ephemeral_metadata_purge_age(ephemeral, BucketConfig) ->
-    %% Purge interval is accepted in # of days but the ep-engine
-    %% needs it to be expressed in seconds.
-    Val = proplists:get_value(purge_interval, BucketConfig,
-                              ?DEFAULT_EPHEMERAL_PURGE_INTERVAL_DAYS),
-    erlang:round(Val * 24 * 3600);
-ephemeral_metadata_purge_age(couchstore, _BucketConfig) ->
-    undefined.
+ephemeral_metadata_purge_age(BucketConfig) ->
+    case ns_bucket:is_persistent(BucketConfig) of
+        true ->
+            undefined;
+        false ->
+            %% Purge interval is accepted in # of days but the ep-engine
+            %% needs it to be expressed in seconds.
+            Val = proplists:get_value(purge_interval, BucketConfig,
+                                      ?DEFAULT_EPHEMERAL_PURGE_INTERVAL_DAYS),
+            erlang:round(Val * 24 * 3600)
+    end.
 
 get(Config, BucketName) ->
     {ok, BucketConfig} = ns_bucket:get_bucket(BucketName, Config),

src/menelaus_stats.erl

@@ -1,5 +1,5 @@
 %% @author Couchbase <info@couchbase.com>
-%% @copyright 2009-2018 Couchbase, Inc.
+%% @copyright 2009-2020 Couchbase, Inc.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -622,7 +622,7 @@ section_nodes(Section) ->
 is_persistent("@"++_) ->
     false;
 is_persistent(BucketName) ->
-    ns_bucket:is_persistent(BucketName).
+    ns_bucket:is_named_bucket_persistent(BucketName).
 
 bucket_exists(Bucket) ->
     ns_bucket:get_bucket(Bucket) =/= not_present.
@@ -2467,7 +2467,7 @@ server_resources_stats_description(ServiceNodes) ->
 
 base_stats_directory(BucketId, ServiceNodes) ->
     {ok, BucketConfig} = ns_bucket:get_bucket(BucketId),
-    Base = case menelaus_web_buckets:external_bucket_type(BucketConfig) of
+    Base = case ns_bucket:external_bucket_type(BucketConfig) of
                membase -> membase_stats_description(BucketId, ServiceNodes);
                memcached -> memcached_stats_description();
                ephemeral -> ephemeral_stats_description(BucketId, ServiceNodes)