wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-05 13:21:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Streaming] Test build fixed (#10617)

Browse Source
This commit is contained in:
Lixin Wei
2020-09-08 14:31:54 +08:00
committed by GitHub
parent ca8792e4ff
commit 2b95e71dac
12 changed files with 87 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streaming/src/test/message_serialization_tests.cc
+3 -3
View File
@@ -16,7 +16,7 @@ TEST(StreamingSerializationTest, streaming_message_serialization_test) {
uint8_t *bytes = new uint8_t[message_length];
message->ToBytes(bytes);
StreamingMessagePtr new_message = StreamingMessage::FromBytes(bytes);
EXPECT_EQ(std::memcmp(new_message->RawData(), data, 3), 0);
EXPECT_EQ(std::memcmp(new_message->Payload(), data, 3), 0);
delete[] bytes;
}
@@ -81,9 +81,9 @@ TEST(StreamingSerializationTest, streaming_message_barrier_bundle_serialization_
EXPECT_TRUE(s_item->ClassBytesSize() == m_item->ClassBytesSize());
EXPECT_TRUE(s_item->GetMessageType() == m_item->GetMessageType());
EXPECT_TRUE(s_item->GetMessageId() == m_item->GetMessageId());
EXPECT_TRUE(s_item->GetDataSize() == m_item->GetDataSize());
EXPECT_TRUE(s_item->PayloadSize() == m_item->PayloadSize());
EXPECT_TRUE(
std::memcmp(s_item->RawData(), m_item->RawData(), m_item->GetDataSize()) == 0);
std::memcmp(s_item->Payload(), m_item->Payload(), m_item->PayloadSize()) == 0);
EXPECT_TRUE(*(s_item.get()) == (*(m_item.get())));
delete[] bundle_bytes;
streaming/src/test/mock_actor.cc
+19 -9
View File
@@ -1,4 +1,5 @@
#define BOOST_BIND_NO_PLACEHOLDERS
#include "common/status.h"
#include "data_reader.h"
#include "data_writer.h"
#include "gtest/gtest.h"
@@ -8,8 +9,7 @@
#include "ray/common/test_util.h"
#include "ray/core_worker/context.h"
#include "ray/core_worker/core_worker.h"
#include "ring_buffer.h"
#include "status.h"
#include "ring_buffer/ring_buffer.h"
using namespace std::placeholders;
const uint32_t MESSAGE_BOUND_SIZE = 10000;
@@ -126,9 +126,13 @@ class StreamingQueueWriterTestSuite : public StreamingQueueTestSuite {
for (uint32_t j = 0; j < buffer_len; ++j) {
data[j] = j % 128;
}
STREAMING_LOG(DEBUG) << "Write data to queue, count=" << i
<< ", queue_id=" << q_id;
writer_client->WriteMessageToBufferRing(q_id, data, buffer_len,
StreamingMessageType::Message);
if (i % 10 == 0) {
writer_client->BroadcastBarrier(i / 10, nullptr, 0);
}
}
++i;
}
@@ -159,7 +163,8 @@ class StreamingQueueReaderTestSuite : public StreamingQueueTestSuite {
for (auto &q_id : queue_id_vec) {
queue_last_cp_id[q_id] = 0;
}
STREAMING_LOG(INFO) << "Start read message bundle";
STREAMING_LOG(INFO) << "Start read message bundle, queue_id_size="
<< queue_id_vec.size();
while (true) {
std::shared_ptr<DataBundle> msg;
StreamingStatus st = reader_client->GetBundle(100, msg);
@@ -173,8 +178,13 @@ class StreamingQueueReaderTestSuite : public StreamingQueueTestSuite {
<< "read null pointer message, queue id => " << msg->from.Hex();
if (msg->meta->GetBundleType() == StreamingMessageBundleType::Barrier) {
STREAMING_LOG(DEBUG) << "barrier message recevied => "
<< msg->meta->GetMessageBundleTs();
StreamingBarrierHeader barrier_header;
StreamingMessage::GetBarrierIdFromRawData(msg->data + kMessageHeaderSize,
&barrier_header);
STREAMING_LOG(DEBUG) << "barrier message recevied, time="
<< msg->meta->GetMessageBundleTs()
<< ", barrier_id=" << barrier_header.barrier_id
<< ", data=" << Util::Byte2hex(msg->data, msg->data_size);
std::unordered_map<ray::ObjectID, ConsumerChannelInfo> *offset_map;
reader_client->GetOffsetInfo(offset_map);
@@ -206,12 +216,12 @@ class StreamingQueueReaderTestSuite : public StreamingQueueTestSuite {
uint32_t buff_len = i % DEFAULT_STREAMING_MESSAGE_BUFFER_SIZE;
if (i > MESSAGE_BOUND_SIZE) break;
EXPECT_EQ(buff_len, item->GetDataSize());
EXPECT_EQ(buff_len, item->PayloadSize());
uint8_t *compared_data = new uint8_t[buff_len];
for (uint32_t j = 0; j < item->GetDataSize(); ++j) {
for (uint32_t j = 0; j < item->PayloadSize(); ++j) {
compared_data[j] = j % 128;
}
EXPECT_EQ(std::memcmp(compared_data, item->RawData(), item->GetDataSize()), 0);
EXPECT_EQ(std::memcmp(compared_data, item->Payload(), item->PayloadSize()), 0);
delete[] compared_data;
}
STREAMING_LOG(DEBUG) << "Received message count => " << recevied_message_cnt;
streaming/src/test/mock_transfer_tests.cc
+4 -4
View File
@@ -77,7 +77,7 @@ TEST_F(StreamingTransferTest, exchange_single_channel_test) {
StreamingMessageBundlePtr bundle_ptr = StreamingMessageBundle::FromBytes(msg->data);
auto &message_list = bundle_ptr->GetMessageList();
auto &message = message_list.front();
EXPECT_EQ(std::memcmp(message->RawData(), data, data_size), 0);
EXPECT_EQ(std::memcmp(message->Payload(), data, data_size), 0);
}
TEST_F(StreamingTransferTest, exchange_multichannel_test) {
@@ -94,7 +94,7 @@ TEST_F(StreamingTransferTest, exchange_multichannel_test) {
StreamingMessageBundlePtr bundle_ptr = StreamingMessageBundle::FromBytes(msg->data);
auto &message_list = bundle_ptr->GetMessageList();
auto &message = message_list.front();
EXPECT_EQ(std::memcmp(message->RawData(), data, data_size), 0);
EXPECT_EQ(std::memcmp(message->Payload(), data, data_size), 0);
}
}
@@ -125,7 +125,7 @@ TEST_F(StreamingTransferTest, exchange_consumed_test) {
int index = 0;
for (auto &message : read_message_list) {
func(index++);
EXPECT_EQ(std::memcmp(message->RawData(), data.get(), data_size), 0);
EXPECT_EQ(std::memcmp(message->Payload(), data.get(), data_size), 0);
}
write_thread.join();
}
@@ -180,7 +180,7 @@ TEST_F(StreamingTransferTest, flow_control_test) {
int index = 0;
for (auto &message : read_message_list) {
func(index++);
EXPECT_EQ(std::memcmp(message->RawData(), data.get(), data_size), 0);
EXPECT_EQ(std::memcmp(message->Payload(), data.get(), data_size), 0);
}
write_thread.join();
}
streaming/src/test/queue_tests_base.h
+1 -9
View File
@@ -87,11 +87,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
args.emplace_back(new TaskArgByValue(std::make_shared<RayObject>(
msg.ToBytes(), nullptr, std::vector<ObjectID>(), true)));
std::unordered_map<std::string, double> resources;
<<<<<<< HEAD
TaskOptions options(0, resources);
=======
TaskOptions options{"", 0, resources};
>>>>>>> 6a78ba9752dc7f17b0e4b7423898c0facf777d3d
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON,
ray::FunctionDescriptorBuilder::BuildPython("", "", "init", "")};
@@ -107,7 +103,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
args.emplace_back(new TaskArgByValue(
std::make_shared<RayObject>(buffer, nullptr, std::vector<ObjectID>(), true)));
std::unordered_map<std::string, double> resources;
TaskOptions options(0, resources);
TaskOptions options("", 0, resources);
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
"", test, "execute_test", "")};
@@ -123,11 +119,7 @@ class StreamingQueueTestBase : public ::testing::TestWithParam<uint64_t> {
args.emplace_back(new TaskArgByValue(
std::make_shared<RayObject>(buffer, nullptr, std::vector<ObjectID>(), true)));
std::unordered_map<std::string, double> resources;
<<<<<<< HEAD
TaskOptions options(1, resources);
=======
TaskOptions options{"", 1, resources};
>>>>>>> 6a78ba9752dc7f17b0e4b7423898c0facf777d3d
std::vector<ObjectID> return_ids;
RayFunction func{ray::Language::PYTHON, ray::FunctionDescriptorBuilder::BuildPython(
"", "", "check_current_test_status", "")};
streaming/src/test/ring_buffer_tests.cc
+5 -5
View File
@@ -5,7 +5,7 @@
#include "gtest/gtest.h"
#include "message/message.h"
#include "ray/util/logging.h"
#include "ring_buffer.h"
#include "ring_buffer/ring_buffer.h"
using namespace ray;
using namespace ray::streaming;
@@ -28,8 +28,8 @@ TEST(StreamingRingBufferTest, streaming_message_ring_buffer_test) {
while (!ring_buffer.IsEmpty()) {
StreamingMessagePtr message_ptr = ring_buffer.Front();
ring_buffer.Pop();
EXPECT_EQ(message_ptr->GetDataSize(), 3);
EXPECT_EQ(*(message_ptr->RawData()), th++);
EXPECT_EQ(message_ptr->PayloadSize(), 3);
EXPECT_EQ(*(message_ptr->Payload()), th++);
}
}
}
@@ -52,7 +52,7 @@ TEST(StreamingRingBufferTest, spsc_test) {
while (ring_buffer.IsEmpty()) {
}
auto &msg = ring_buffer.Front();
EXPECT_EQ(std::memcmp(msg->RawData(), &count, sizeof(size_t)), 0);
EXPECT_EQ(std::memcmp(msg->Payload(), &count, sizeof(size_t)), 0);
ring_buffer.Pop();
count++;
}
@@ -78,7 +78,7 @@ TEST(StreamingRingBufferTest, mutex_test) {
while (ring_buffer.IsEmpty()) {
}
auto msg = ring_buffer.Front();
EXPECT_EQ(std::memcmp(msg->RawData(), &count, sizeof(size_t)), 0);
EXPECT_EQ(std::memcmp(msg->Payload(), &count, sizeof(size_t)), 0);
ring_buffer.Pop();
count++;
}
streaming/src/test/run_streaming_queue_test.sh
+2
View File
@@ -53,7 +53,9 @@ STREAMING_TEST_WORKER_EXEC="$RAY_ROOT/bazel-bin/streaming/streaming_test_worker"
GCS_SERVER_EXEC="$RAY_ROOT/bazel-bin/gcs_server"
# clear env
set +e
pgrep "plasma|DefaultDriver|DefaultWorker|AppStarter|redis|http_server|job_agent" | xargs kill -9 &> /dev/null
set -e
# Run tests.
streaming/src/test/streaming_queue_tests.cc
+1 -1
View File
@@ -7,7 +7,7 @@
#include "queue/queue_client.h"
#include "ray/common/test_util.h"
#include "ray/core_worker/core_worker.h"
#include "ring_buffer.h"
#include "ring_buffer/ring_buffer.h"
#include "test/queue_tests_base.h"
using namespace std::placeholders;