MB-19897: Fix the data race on lastSendTime between stats and dcp worker threads

Fix the thread sanitizer warning
http://cv.jenkins.couchbase.com/job/ep-engine-threadsanitizer-3.0.x/258/console

WARNING: ThreadSanitizer: data race (pid=102290)
  Read of size 4 at 0x7d580000f71c by thread T14 (mutexes: write M969):
    #0 void ConnHandler::addStat<unsigned int>(char const*, unsigned int const&, void (*)(char const*, unsigned short, char const*, unsigned int, void const*), void const*) const /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/tapconnection.h:294 (ep.so+0x00000020e9f3)
    #1 DcpProducer::addStats(void (*)(char const*, unsigned short, char const*, unsigned int, void const*), void const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-producer.cc:557 (ep.so+0x00000027fbe5)
    #2 ConnStatBuilder::operator()(SingleThreadedRCPtr<ConnHandler>&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:3701 (ep.so+0x000000183c44)
    #3 ConnStatBuilder std::for_each<std::_List_iterator<SingleThreadedRCPtr<ConnHandler> >, ConnStatBuilder>(std::_List_iterator<SingleThreadedRCPtr<ConnHandler> >, std::_List_iterator<SingleThreadedRCPtr<ConnHandler> >, ConnStatBuilder) /usr/bin/../lib/gcc/x86_64-linux-gnu/4.9/../../../../include/c++/4.9/bits/stl_algo.h:3755 (ep.so+0x0000001838a5)
    #4 void ConnMap::each_UNLOCKED<ConnStatBuilder>(ConnStatBuilder) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/connmap.h:148 (ep.so+0x000000183808)
    #5 void ConnMap::each<ConnStatBuilder>(ConnStatBuilder) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/connmap.h:140 (ep.so+0x00000017732e)
    #6 EventuallyPersistentEngine::doDcpStats(void const*, void (*)(char const*, unsigned short, char const*, unsigned int, void const*)) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:3954 (ep.so+0x00000014c5ed)

  Previous write of size 4 at 0x7d580000f71c by main thread:
    #0 DcpProducer::maybeSendNoop(dcp_message_producers*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-producer.cc:740 (ep.so+0x00000027d8ce)
    #1 DcpProducer::step(dcp_message_producers*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-producer.cc:323 (ep.so+0x00000027c920)
    #2 EvpDcpStep(engine_interface*, void const*, dcp_message_producers*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:1404 (ep.so+0x000000138baa)

Change-Id: I2a2b0b0f01b10ecb31701bfc2330881bbafc6b74
Reviewed-on: http://review.couchbase.org/64959
Well-Formed: buildbot <build@couchbase.com>
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Dave Rigby <daver@couchbase.com>

Author: manudhundi

src/dcp-producer.cc

@@ -442,7 +442,7 @@ ENGINE_ERROR_CODE DcpProducer::step(struct dcp_message_producers* producers) {
         delete resp;
     }
 
-    lastSendTime = ep_current_time();
+    lastSendTime.store(ep_current_time(), std::memory_order_relaxed);
     return (ret == ENGINE_SUCCESS) ? ENGINE_WANT_MORE : ret;
 }
 
@@ -608,7 +608,8 @@ void DcpProducer::addStats(ADD_STAT add_stat, const void *c) {
     addStat("items_sent", getItemsSent(), add_stat, c);
     addStat("items_remaining", getItemsRemaining(), add_stat, c);
     addStat("total_bytes_sent", getTotalBytes(), add_stat, c);
-    addStat("last_sent_time", lastSendTime, add_stat, c);
+    addStat("last_sent_time", lastSendTime.load(std::memory_order_relaxed),
+            add_stat, c);
     addStat("noop_enabled", noopCtx.enabled, add_stat, c);
     addStat("noop_wait", noopCtx.pendingRecv, add_stat, c);
     addStat("priority", priority.c_str(), add_stat, c);
@@ -798,7 +799,8 @@ ENGINE_ERROR_CODE DcpProducer::maybeSendNoop(struct dcp_message_producers* produ
                 ret = ENGINE_WANT_MORE;
                 noopCtx.pendingRecv = true;
                 noopCtx.sendTime = ep_current_time();
-                lastSendTime = ep_current_time();
+                lastSendTime.store(ep_current_time(),
+                                   std::memory_order_relaxed);
             }
             return ret;
         }

src/dcp-producer.h

@@ -307,7 +307,7 @@ class DcpProducer : public Producer {
 
     bool notifyOnly;
     bool enableExtMetaData;
-    rel_time_t lastSendTime;
+    AtomicValue<rel_time_t> lastSendTime;
     BufferLog log;
 
     BackfillManager* backfillMgr;