mirror of
https://github.com/wassname/ray.git
synced 2026-07-01 19:14:35 +08:00
Lint Cython files (#4097)
This commit is contained in:
Philipp Moritz
committed by
Robert Nishihara
Robert Nishihara
parent
acf4d53b55
commit
bcd5af78c7
@@ -14,29 +14,37 @@ from ray.includes.unique_ids cimport (
|
||||
|
||||
cdef extern from "ray/status.h" namespace "ray" nogil:
|
||||
cdef cppclass StatusCode:
|
||||
pass
|
||||
pass
|
||||
|
||||
cdef cppclass CRayStatus "ray::Status":
|
||||
RayStatus()
|
||||
RayStatus(StatusCode code, const c_string &msg)
|
||||
RayStatus(const CRayStatus &s);
|
||||
RayStatus(const CRayStatus &s)
|
||||
|
||||
@staticmethod
|
||||
CRayStatus OK()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus OutOfMemory()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus KeyError()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus Invalid()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus IOError()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus TypeError()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus UnknownError()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus NotImplemented()
|
||||
|
||||
@staticmethod
|
||||
CRayStatus RedisError()
|
||||
|
||||
@@ -75,12 +83,12 @@ cdef extern from "ray/status.h" namespace "ray::StatusCode" nogil:
|
||||
cdef extern from "ray/id.h" namespace "ray" nogil:
|
||||
const CTaskID FinishTaskId(const CTaskID &task_id)
|
||||
const CObjectID ComputeReturnId(const CTaskID &task_id,
|
||||
int64_t return_index)
|
||||
int64_t return_index)
|
||||
const CObjectID ComputePutId(const CTaskID &task_id, int64_t put_index)
|
||||
const CTaskID ComputeTaskId(const CObjectID &object_id)
|
||||
const CTaskID GenerateTaskId(const CDriverID &driver_id,
|
||||
const CTaskID &parent_task_id,
|
||||
int parent_task_counter)
|
||||
const CTaskID &parent_task_id,
|
||||
int parent_task_counter)
|
||||
int64_t ComputeObjectIndex(const CObjectID &object_id)
|
||||
|
||||
|
||||
@@ -92,18 +100,21 @@ cdef extern from "ray/gcs/format/gcs_generated.h" nogil:
|
||||
pass
|
||||
|
||||
|
||||
# This is a workaround for C++ enum class since Cython has no corresponding representation.
|
||||
# This is a workaround for C++ enum class since Cython has no corresponding
|
||||
# representation.
|
||||
cdef extern from "ray/gcs/format/gcs_generated.h" namespace "Language" nogil:
|
||||
cdef CLanguage LANGUAGE_PYTHON "Language::PYTHON"
|
||||
cdef CLanguage LANGUAGE_CPP "Language::CPP"
|
||||
cdef CLanguage LANGUAGE_JAVA "Language::JAVA"
|
||||
|
||||
|
||||
cdef extern from "ray/raylet/scheduling_resources.h" namespace "ray::raylet" nogil:
|
||||
cdef extern from "ray/raylet/scheduling_resources.h" \
|
||||
namespace "ray::raylet" nogil:
|
||||
cdef cppclass ResourceSet "ResourceSet":
|
||||
ResourceSet()
|
||||
ResourceSet(const unordered_map[c_string, double] &resource_map)
|
||||
ResourceSet(const c_vector[c_string] &resource_labels, const c_vector[double] resource_capacity)
|
||||
ResourceSet(const c_vector[c_string] &resource_labels,
|
||||
const c_vector[double] resource_capacity)
|
||||
c_bool operator==(const ResourceSet &rhs) const
|
||||
c_bool IsEqual(const ResourceSet &other) const
|
||||
c_bool IsSubset(const ResourceSet &other) const
|
||||
|
||||
@@ -37,7 +37,8 @@ cdef extern from "ray/gcs/format/gcs_generated.h" nogil:
|
||||
GCSProfileTableDataT()
|
||||
|
||||
|
||||
ctypedef unordered_map[c_string, c_vector[pair[int64_t, double]]] ResourceMappingType
|
||||
ctypedef unordered_map[c_string, c_vector[pair[int64_t, double]]] \
|
||||
ResourceMappingType
|
||||
ctypedef pair[c_vector[CObjectID], c_vector[CObjectID]] WaitResultPair
|
||||
|
||||
|
||||
@@ -48,22 +49,25 @@ cdef extern from "ray/raylet/raylet_client.h" nogil:
|
||||
c_bool is_worker, const CDriverID &driver_id,
|
||||
const CLanguage &language)
|
||||
CRayStatus Disconnect()
|
||||
CRayStatus SubmitTask(const c_vector[CObjectID] &execution_dependencies,
|
||||
const CTaskSpecification &task_spec)
|
||||
CRayStatus SubmitTask(
|
||||
const c_vector[CObjectID] &execution_dependencies,
|
||||
const CTaskSpecification &task_spec)
|
||||
CRayStatus GetTask(unique_ptr[CTaskSpecification] *task_spec)
|
||||
CRayStatus TaskDone()
|
||||
CRayStatus FetchOrReconstruct(c_vector[CObjectID] &object_ids,
|
||||
c_bool fetch_only,
|
||||
const CTaskID ¤t_task_id)
|
||||
c_bool fetch_only,
|
||||
const CTaskID ¤t_task_id)
|
||||
CRayStatus NotifyUnblocked(const CTaskID ¤t_task_id)
|
||||
CRayStatus Wait(const c_vector[CObjectID] &object_ids, int num_returns,
|
||||
int64_t timeout_milliseconds, c_bool wait_local,
|
||||
const CTaskID ¤t_task_id, WaitResultPair *result)
|
||||
CRayStatus Wait(const c_vector[CObjectID] &object_ids,
|
||||
int num_returns, int64_t timeout_milliseconds,
|
||||
c_bool wait_local, const CTaskID ¤t_task_id,
|
||||
WaitResultPair *result)
|
||||
CRayStatus PushError(const CDriverID &job_id, const c_string &type,
|
||||
const c_string &error_message, double timestamp)
|
||||
CRayStatus PushProfileEvents(const GCSProfileTableDataT &profile_events)
|
||||
CRayStatus PushProfileEvents(
|
||||
const GCSProfileTableDataT &profile_events)
|
||||
CRayStatus FreeObjects(const c_vector[CObjectID] &object_ids,
|
||||
c_bool local_only)
|
||||
c_bool local_only)
|
||||
CRayStatus PrepareActorCheckpoint(const CActorID &actor_id,
|
||||
CActorCheckpointID &checkpoint_id)
|
||||
CRayStatus NotifyActorResumedFromCheckpoint(
|
||||
|
||||
@@ -27,7 +27,8 @@ cdef class Config:
|
||||
|
||||
@staticmethod
|
||||
def initial_reconstruction_timeout_milliseconds():
|
||||
return RayConfig.instance().initial_reconstruction_timeout_milliseconds()
|
||||
return (RayConfig.instance()
|
||||
.initial_reconstruction_timeout_milliseconds())
|
||||
|
||||
@staticmethod
|
||||
def get_timeout_milliseconds():
|
||||
@@ -59,11 +60,13 @@ cdef class Config:
|
||||
|
||||
@staticmethod
|
||||
def local_scheduler_fetch_timeout_milliseconds():
|
||||
return RayConfig.instance().local_scheduler_fetch_timeout_milliseconds()
|
||||
return (RayConfig.instance()
|
||||
.local_scheduler_fetch_timeout_milliseconds())
|
||||
|
||||
@staticmethod
|
||||
def local_scheduler_reconstruction_timeout_milliseconds():
|
||||
return RayConfig.instance().local_scheduler_reconstruction_timeout_milliseconds()
|
||||
return (RayConfig.instance()
|
||||
.local_scheduler_reconstruction_timeout_milliseconds())
|
||||
|
||||
@staticmethod
|
||||
def max_num_to_reconstruct():
|
||||
@@ -119,7 +122,8 @@ cdef class Config:
|
||||
|
||||
@staticmethod
|
||||
def node_manager_forward_task_retry_timeout_milliseconds():
|
||||
return RayConfig.instance().node_manager_forward_task_retry_timeout_milliseconds()
|
||||
return (RayConfig.instance()
|
||||
.node_manager_forward_task_retry_timeout_milliseconds())
|
||||
|
||||
@staticmethod
|
||||
def object_manager_pull_timeout_ms():
|
||||
|
||||
@@ -18,10 +18,13 @@ from ray.includes.unique_ids cimport (
|
||||
)
|
||||
|
||||
|
||||
cdef extern from "ray/raylet/task_execution_spec.h" namespace "ray::raylet" nogil:
|
||||
cdef cppclass CTaskExecutionSpecification "ray::raylet::TaskExecutionSpecification":
|
||||
cdef extern from "ray/raylet/task_execution_spec.h" \
|
||||
namespace "ray::raylet" nogil:
|
||||
cdef cppclass CTaskExecutionSpecification \
|
||||
"ray::raylet::TaskExecutionSpecification":
|
||||
CTaskExecutionSpecification(const c_vector[CObjectID] &&dependencies)
|
||||
CTaskExecutionSpecification(const c_vector[CObjectID] &&dependencies, int num_forwards)
|
||||
CTaskExecutionSpecification(
|
||||
const c_vector[CObjectID] &&dependencies, int num_forwards)
|
||||
c_vector[CObjectID] ExecutionDependencies() const
|
||||
void SetExecutionDependencies(const c_vector[CObjectID] &dependencies)
|
||||
int NumForwards() const
|
||||
@@ -34,31 +37,35 @@ cdef extern from "ray/raylet/task_spec.h" namespace "ray::raylet" nogil:
|
||||
cdef cppclass CTaskArgument "ray::raylet::TaskArgument":
|
||||
pass
|
||||
|
||||
cdef cppclass CTaskArgumentByReference "ray::raylet::TaskArgumentByReference":
|
||||
CTaskArgumentByReference(const c_vector[CObjectID] &references);
|
||||
cdef cppclass CTaskArgumentByReference \
|
||||
"ray::raylet::TaskArgumentByReference":
|
||||
CTaskArgumentByReference(const c_vector[CObjectID] &references)
|
||||
|
||||
cdef cppclass CTaskArgumentByValue "ray::raylet::TaskArgumentByValue":
|
||||
CTaskArgumentByValue(const uint8_t *value, size_t length);
|
||||
CTaskArgumentByValue(const uint8_t *value, size_t length)
|
||||
|
||||
cdef cppclass CTaskSpecification "ray::raylet::TaskSpecification":
|
||||
CTaskSpecification(const CDriverID &driver_id, const CTaskID &parent_task_id,
|
||||
int64_t parent_counter,
|
||||
const c_vector[shared_ptr[CTaskArgument]] &task_arguments,
|
||||
int64_t num_returns,
|
||||
const unordered_map[c_string, double] &required_resources,
|
||||
const CLanguage &language,
|
||||
const c_vector[c_string] &function_descriptor)
|
||||
CTaskSpecification(
|
||||
const CDriverID &driver_id, const CTaskID &parent_task_id, int64_t parent_counter,
|
||||
const CActorID &actor_creation_id, const CObjectID &actor_creation_dummy_object_id,
|
||||
const CDriverID &driver_id, const CTaskID &parent_task_id,
|
||||
int64_t parent_counter,
|
||||
const c_vector[shared_ptr[CTaskArgument]] &task_arguments,
|
||||
int64_t num_returns,
|
||||
const unordered_map[c_string, double] &required_resources,
|
||||
const CLanguage &language,
|
||||
const c_vector[c_string] &function_descriptor)
|
||||
CTaskSpecification(
|
||||
const CDriverID &driver_id, const CTaskID &parent_task_id,
|
||||
int64_t parent_counter, const CActorID &actor_creation_id,
|
||||
const CObjectID &actor_creation_dummy_object_id,
|
||||
int64_t max_actor_reconstructions, const CActorID &actor_id,
|
||||
const CActorHandleID &actor_handle_id, int64_t actor_counter,
|
||||
const c_vector[CActorHandleID] &new_actor_handles,
|
||||
const c_vector[shared_ptr[CTaskArgument]] &task_arguments,
|
||||
int64_t num_returns,
|
||||
const unordered_map[c_string, double] &required_resources,
|
||||
const unordered_map[c_string, double] &required_placement_resources,
|
||||
const CLanguage &language, const c_vector[c_string] &function_descriptor)
|
||||
const unordered_map[c_string, double] &required_placement_res,
|
||||
const CLanguage &language,
|
||||
const c_vector[c_string] &function_descriptor)
|
||||
CTaskSpecification(const c_string &string)
|
||||
c_string SerializeAsString() const
|
||||
|
||||
@@ -97,7 +104,7 @@ cdef extern from "ray/raylet/task_spec.h" namespace "ray::raylet" nogil:
|
||||
cdef extern from "ray/raylet/task.h" namespace "ray::raylet" nogil:
|
||||
cdef cppclass CTask "ray::raylet::Task":
|
||||
CTask(const CTaskExecutionSpecification &execution_spec,
|
||||
const CTaskSpecification &task_spec)
|
||||
const CTaskSpecification &task_spec)
|
||||
const CTaskExecutionSpecification &GetTaskExecutionSpec() const
|
||||
const CTaskSpecification &GetTaskSpecification() const
|
||||
void SetExecutionDependencies(const c_vector[CObjectID] &dependencies)
|
||||
@@ -105,5 +112,6 @@ cdef extern from "ray/raylet/task.h" namespace "ray::raylet" nogil:
|
||||
const c_vector[CObjectID] &GetDependencies() const
|
||||
void CopyTaskExecutionSpec(const CTask &task)
|
||||
|
||||
cdef c_string SerializeTaskAsString(const c_vector[CObjectID] *dependencies,
|
||||
cdef c_string SerializeTaskAsString(
|
||||
const c_vector[CObjectID] *dependencies,
|
||||
const CTaskSpecification *task_spec)
|
||||
|
||||
@@ -37,42 +37,57 @@ cdef class Task:
|
||||
|
||||
for item in function_descriptor:
|
||||
if not isinstance(item, bytes):
|
||||
raise TypeError("'function_descriptor' takes a list of byte strings.")
|
||||
raise TypeError(
|
||||
"'function_descriptor' takes a list of byte strings.")
|
||||
c_function_descriptor.push_back(item)
|
||||
|
||||
# Parse the resource map.
|
||||
if resource_map is not None:
|
||||
required_resources = resource_map_from_python_dict(resource_map)
|
||||
required_resources = resource_map_from_dict(resource_map)
|
||||
if required_resources.count(b"CPU") == 0:
|
||||
required_resources[b"CPU"] = 1.0
|
||||
if placement_resource_map is not None:
|
||||
required_placement_resources = resource_map_from_python_dict(placement_resource_map)
|
||||
required_placement_resources = (
|
||||
resource_map_from_dict(placement_resource_map))
|
||||
|
||||
# Parse the arguments from the list.
|
||||
for arg in arguments:
|
||||
if isinstance(arg, ObjectID):
|
||||
references = c_vector[CObjectID]()
|
||||
references.push_back((<ObjectID>arg).data)
|
||||
task_args.push_back(static_pointer_cast[CTaskArgument, CTaskArgumentByReference](make_shared[CTaskArgumentByReference](references)))
|
||||
task_args.push_back(
|
||||
static_pointer_cast[CTaskArgument,
|
||||
CTaskArgumentByReference](
|
||||
make_shared[CTaskArgumentByReference](references)))
|
||||
else:
|
||||
pickled_str = pickle.dumps(arg, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
task_args.push_back(static_pointer_cast[CTaskArgument, CTaskArgumentByValue](make_shared[CTaskArgumentByValue](<uint8_t *>pickled_str.c_str(), pickled_str.size())))
|
||||
pickled_str = pickle.dumps(
|
||||
arg, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
task_args.push_back(
|
||||
static_pointer_cast[CTaskArgument,
|
||||
CTaskArgumentByValue](
|
||||
make_shared[CTaskArgumentByValue](
|
||||
<uint8_t *>pickled_str.c_str(),
|
||||
pickled_str.size())))
|
||||
|
||||
for new_actor_handle in new_actor_handles:
|
||||
task_new_actor_handles.push_back((<ActorHandleID?>new_actor_handle).data)
|
||||
task_new_actor_handles.push_back(
|
||||
(<ActorHandleID?>new_actor_handle).data)
|
||||
|
||||
self.task_spec.reset(new CTaskSpecification(
|
||||
CUniqueID(driver_id.data), parent_task_id.data, parent_counter, actor_creation_id.data,
|
||||
actor_creation_dummy_object_id.data, max_actor_reconstructions, CUniqueID(actor_id.data),
|
||||
CUniqueID(actor_handle_id.data), actor_counter, task_new_actor_handles, task_args, num_returns,
|
||||
required_resources, required_placement_resources, LANGUAGE_PYTHON,
|
||||
c_function_descriptor))
|
||||
CUniqueID(driver_id.data), parent_task_id.data, parent_counter,
|
||||
actor_creation_id.data, actor_creation_dummy_object_id.data,
|
||||
max_actor_reconstructions, CUniqueID(actor_id.data),
|
||||
CUniqueID(actor_handle_id.data), actor_counter,
|
||||
task_new_actor_handles, task_args, num_returns,
|
||||
required_resources, required_placement_resources,
|
||||
LANGUAGE_PYTHON, c_function_descriptor))
|
||||
|
||||
# Set the task's execution dependencies.
|
||||
self.execution_dependencies.reset(new c_vector[CObjectID]())
|
||||
if execution_arguments is not None:
|
||||
for execution_arg in execution_arguments:
|
||||
self.execution_dependencies.get().push_back((<ObjectID?>execution_arg).data)
|
||||
self.execution_dependencies.get().push_back(
|
||||
(<ObjectID?>execution_arg).data)
|
||||
|
||||
@staticmethod
|
||||
cdef make(unique_ptr[CTaskSpecification]& task_spec):
|
||||
@@ -108,7 +123,8 @@ cdef class Task:
|
||||
return self.task_spec.get().SerializeAsString()
|
||||
|
||||
def _serialized_raylet_task(self):
|
||||
return SerializeTaskAsString(self.execution_dependencies.get(), self.task_spec.get())
|
||||
return SerializeTaskAsString(
|
||||
self.execution_dependencies.get(), self.task_spec.get())
|
||||
|
||||
def driver_id(self):
|
||||
"""Return the driver ID for this task."""
|
||||
@@ -128,7 +144,8 @@ cdef class Task:
|
||||
|
||||
def function_descriptor_list(self):
|
||||
"""Return the function descriptor for this task."""
|
||||
cdef c_vector[c_string] function_descriptor = self.task_spec.get().FunctionDescriptor()
|
||||
cdef c_vector[c_string] function_descriptor = (
|
||||
self.task_spec.get().FunctionDescriptor())
|
||||
results = []
|
||||
for i in range(function_descriptor.size()):
|
||||
results.append(function_descriptor[i])
|
||||
@@ -148,9 +165,11 @@ cdef class Task:
|
||||
count = task_spec.ArgIdCount(i)
|
||||
if count > 0:
|
||||
assert count == 1
|
||||
arg_list.append(ObjectID.from_native(task_spec.ArgId(i, 0)))
|
||||
arg_list.append(
|
||||
ObjectID.from_native(task_spec.ArgId(i, 0)))
|
||||
else:
|
||||
serialized_str = task_spec.ArgVal(i)[:task_spec.ArgValLength(i)]
|
||||
serialized_str = (
|
||||
task_spec.ArgVal(i)[:task_spec.ArgValLength(i)])
|
||||
obj = pickle.loads(serialized_str)
|
||||
arg_list.append(obj)
|
||||
elif lang == <int32_t>LANGUAGE_JAVA:
|
||||
@@ -169,15 +188,18 @@ cdef class Task:
|
||||
def required_resources(self):
|
||||
"""Return the resource dictionary of the task."""
|
||||
cdef:
|
||||
unordered_map[c_string, double] resource_map = self.task_spec.get().GetRequiredResources().GetResourceMap()
|
||||
unordered_map[c_string, double] resource_map = (
|
||||
self.task_spec.get().GetRequiredResources().GetResourceMap())
|
||||
c_string resource_name
|
||||
double resource_value
|
||||
unordered_map[c_string, double].iterator iterator = resource_map.begin()
|
||||
unordered_map[c_string, double].iterator iterator = (
|
||||
resource_map.begin())
|
||||
|
||||
required_resources = {}
|
||||
while iterator != resource_map.end():
|
||||
resource_name = dereference(iterator).first
|
||||
py_resource_name = str(resource_name) # bytes for Py2, unicode for Py3
|
||||
# bytes for Py2, unicode for Py3
|
||||
py_resource_name = str(resource_name)
|
||||
resource_value = dereference(iterator).second
|
||||
required_resources[py_resource_name] = resource_value
|
||||
postincrement(iterator)
|
||||
@@ -193,7 +215,8 @@ cdef class Task:
|
||||
|
||||
def actor_creation_dummy_object_id(self):
|
||||
"""Return the actor creation dummy object ID for the task."""
|
||||
return ObjectID.from_native(self.task_spec.get().ActorCreationDummyObjectId())
|
||||
return ObjectID.from_native(
|
||||
self.task_spec.get().ActorCreationDummyObjectId())
|
||||
|
||||
def actor_id(self):
|
||||
"""Return the actor ID for this task."""
|
||||
|
||||
@@ -6,21 +6,25 @@ cdef extern from "ray/id.h" namespace "ray" nogil:
|
||||
cdef cppclass CUniqueID "ray::UniqueID":
|
||||
CUniqueID()
|
||||
CUniqueID(const CUniqueID &from_id)
|
||||
|
||||
@staticmethod
|
||||
CUniqueID from_random()
|
||||
|
||||
@staticmethod
|
||||
CUniqueID from_binary(const c_string & binary)
|
||||
|
||||
@staticmethod
|
||||
const CUniqueID nil()
|
||||
|
||||
size_t hash() const
|
||||
c_bool is_nil() const
|
||||
c_bool operator==(const CUniqueID& rhs) const
|
||||
c_bool operator!=(const CUniqueID& rhs) const
|
||||
const uint8_t *data() const
|
||||
uint8_t *mutable_data();
|
||||
size_t size() const;
|
||||
c_string binary() const;
|
||||
c_string hex() const;
|
||||
uint8_t *mutable_data()
|
||||
size_t size() const
|
||||
c_string binary() const
|
||||
c_string hex() const
|
||||
|
||||
ctypedef CUniqueID CActorCheckpointID
|
||||
ctypedef CUniqueID CActorClassID
|
||||
|
||||
@@ -32,7 +32,8 @@ def check_id(b):
|
||||
if not isinstance(b, bytes):
|
||||
raise TypeError("Unsupported type: " + str(type(b)))
|
||||
if len(b) != kUniqueIDSize:
|
||||
raise ValueError("ID string needs to have length " + str(kUniqueIDSize))
|
||||
raise ValueError("ID string needs to have length " +
|
||||
str(kUniqueIDSize))
|
||||
|
||||
|
||||
cdef extern from "ray/constants.h" nogil:
|
||||
@@ -106,9 +107,10 @@ cdef class UniqueID:
|
||||
return type(self), (self.binary(),)
|
||||
|
||||
def redis_shard_hash(self):
|
||||
# NOTE: The hash function used here must match the one in GetRedisContext in
|
||||
# src/ray/gcs/tables.h. Changes to the hash function should only be made
|
||||
# through std::hash in src/common/common.h
|
||||
# NOTE: The hash function used here must match the one in
|
||||
# GetRedisContext in src/ray/gcs/tables.h. Changes to the
|
||||
# hash function should only be made through std::hash in
|
||||
# src/common/common.h
|
||||
return self.data.hash()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user