MB-64910 MB-61292: Fix scenario when bucket is...

... not present in 'servers' list in bucket props but still
can be used in future (so we should maintain DEKs for it).

The scenario is the following:
1. Create 3 node cluster
2. Shutdown one node (node#3), and let the cluster fail it over
3. Start node#3 again
4. Add node#3 back using delta recovery

At step #3 node#3 will still have bucket data (including DEKs) on
disk while bucket props will not have node#3 in 'servers' list.
In this scenario cb_cluster_secrets should continue to maintain
bucket DEKs.

Change-Id: I51d568bbd92423225f85b9c335dad4030197fde3
Reviewed-on: https://review.couchbase.org/c/ns_server/+/223328
Tested-by: Timofey Barmin <timofey.barmin@couchbase.com>
Reviewed-by: Navdeep S Boparai <navdeep.boparai@couchbase.com>
Well-Formed: Build Bot <build@couchbase.com>

Author: timofey-barmin

apps/ns_server/src/cb_cluster_secrets.erl

@@ -711,6 +711,7 @@ init([Type]) ->
                 case Type == ?NODE_PROC andalso
                      cb_deks:dek_chronicle_keys_filter(Key) of
                     false -> false;
+                    [] -> false;
                     List -> {true, {dek_settings_updated, List}}
                 end
         end,

apps/ns_server/src/cb_deks.erl

@@ -244,12 +244,21 @@ increment_counter_in_chronicle(Kind, SecretId) ->
 dek_chronicle_keys_filter(?CHRONICLE_ENCR_AT_REST_SETTINGS_KEY) ->
     [configDek, logDek, auditDek];
 dek_chronicle_keys_filter(Key) ->
-    case ns_bucket:sub_key_match(Key) of
-        {true, Bucket, props} -> [{bucketDek, Bucket}];
-        {true, Bucket, encr_at_rest} -> [{bucketDek, Bucket}];
-        {true, _Bucket, _} -> false;
-        false -> false
-    end.
+    MembershipKeys = ns_cluster_membership:node_membership_keys(node()),
+    lists:uniq(
+        case ns_bucket:sub_key_match(Key) of
+            {true, Bucket, props} -> [{bucketDek, Bucket}];
+            {true, Bucket, encr_at_rest} -> [{bucketDek, Bucket}];
+            {true, _Bucket, _} -> [];
+            false -> []
+        end ++
+        case lists:member(Key, MembershipKeys) of
+            true ->
+                Buckets = ns_bucket:get_bucket_names(direct),
+                [{bucketDek, B} || B <- Buckets];
+            false ->
+                []
+        end).
 
 %% encryption_method_callback - called to determine if encryption is enabled
 %% or not for that type of entity.
@@ -346,9 +355,11 @@ dek_config({bucketDek, Bucket}) ->
                                      ns_memcached:get_dek_ids_in_use(Bucket)
                                  end,
       drop_callback => drop_bucket_deks(Bucket, _),
-      chronicle_txn_keys => [ns_bucket:root(),
-                             ns_bucket:sub_key(Bucket, props),
-                             ns_bucket:sub_key(Bucket, encr_at_rest)],
+      chronicle_txn_keys =>
+          [ns_bucket:root(),
+           ns_bucket:sub_key(Bucket, props),
+           ns_bucket:sub_key(Bucket, encr_at_rest) |
+           ns_cluster_membership:node_membership_keys(node())],
       required_usage => {bucket_encryption, Bucket}}.
 
 %% Returns all possible deks kinds on the node.

apps/ns_server/src/ns_bucket.erl

@@ -2580,7 +2580,12 @@ validate_encryption_secret(SecretId, Bucket, Snapshot) ->
 get_encryption(BucketName, Snapshot) ->
     case get_bucket(BucketName, Snapshot) of
         {ok, BucketConfig} ->
-            case lists:member(node(), get_servers(BucketConfig)) of
+            IsNodeInServers = lists:member(node(), get_servers(BucketConfig)),
+            {ok, Dir} = ns_storage_conf:this_node_bucket_dbdir(BucketName),
+            ExistsOnDisk = filelib:is_dir(Dir),
+            Services = ns_cluster_membership:node_services(Snapshot, node()),
+            IsKVNode = lists:member(kv, Services),
+            case IsNodeInServers orelse (IsKVNode andalso ExistsOnDisk) of
                 true ->
                     case proplists:get_value(encryption_secret_id, BucketConfig,
                                              ?SECRET_ID_NOT_SET) of

apps/ns_server/src/ns_cluster_membership.erl

@@ -55,7 +55,8 @@
          attach_node_uuids/2,
          fetch_snapshot/1,
          get_snapshot/1,
-         get_snapshot/0
+         get_snapshot/0,
+         node_membership_keys/1
         ]).
 
 -export([supported_services/0,