mirror of
https://github.com/wassname/ray.git
synced 2026-08-01 12:51:09 +08:00
[Streaming]Format java code using IDEA (#10440)
This commit is contained in:
+5
-6
@@ -21,21 +21,20 @@ package io.ray.streaming.state;
|
||||
/**
|
||||
* TransactionState interface.
|
||||
* <p>
|
||||
* Streaming State should implement transaction in case of failure,
|
||||
* which in our case is four default method, finish, commit, ackCommit, rollback.
|
||||
* Streaming State should implement transaction in case of failure, which in our case is four
|
||||
* default method, finish, commit, ackCommit, rollback.
|
||||
*/
|
||||
public interface StateStoreManager {
|
||||
|
||||
/**
|
||||
* The finish method is used when the batched data is all saved in state.
|
||||
* Normally, serialization job is done here.
|
||||
* The finish method is used when the batched data is all saved in state. Normally, serialization
|
||||
* job is done here.
|
||||
*/
|
||||
void finish(long checkpointId);
|
||||
|
||||
/**
|
||||
* The commit method is used for persistent, and can be used in another thread to reach async
|
||||
* state commit.
|
||||
* Normally, data persistent is done here.
|
||||
* state commit. Normally, data persistent is done here.
|
||||
*/
|
||||
void commit(long checkpointId);
|
||||
|
||||
|
||||
+1
@@ -20,6 +20,7 @@ package io.ray.streaming.state.backend;
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import io.ray.streaming.state.config.ConfigKey;
|
||||
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
|
||||
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
|
||||
|
||||
+5
-4
@@ -31,16 +31,17 @@ import io.ray.streaming.state.keystate.state.proxy.MapStateStoreManagerProxy;
|
||||
import io.ray.streaming.state.keystate.state.proxy.ValueStateStoreManagerProxy;
|
||||
|
||||
/**
|
||||
* key state backend manager, managing different kinds of states in different thread.
|
||||
* This class is not thread safe.
|
||||
* key state backend manager, managing different kinds of states in different thread. This class is
|
||||
* not thread safe.
|
||||
*/
|
||||
public class KeyStateBackend extends AbstractKeyStateBackend {
|
||||
|
||||
protected final int numberOfKeyGroups;
|
||||
protected final KeyGroup keyGroup;
|
||||
|
||||
public KeyStateBackend(int numberOfKeyGroups, KeyGroup keyGroup,
|
||||
AbstractStateBackend abstractStateBackend) {
|
||||
public KeyStateBackend(
|
||||
int numberOfKeyGroups, KeyGroup keyGroup,
|
||||
AbstractStateBackend abstractStateBackend) {
|
||||
super(abstractStateBackend);
|
||||
this.numberOfKeyGroups = numberOfKeyGroups;
|
||||
this.keyGroup = keyGroup;
|
||||
|
||||
+3
-2
@@ -27,8 +27,9 @@ import java.util.Map;
|
||||
*/
|
||||
public class StateBackendBuilder {
|
||||
|
||||
private static AbstractStateBackend getStateBackend(Map<String, String> config,
|
||||
BackendType type) {
|
||||
private static AbstractStateBackend getStateBackend(
|
||||
Map<String, String> config,
|
||||
BackendType type) {
|
||||
switch (type) {
|
||||
case MEMORY:
|
||||
return new MemoryStateBackend(config);
|
||||
|
||||
+1
-2
@@ -21,8 +21,7 @@ package io.ray.streaming.state.config;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Config Helper figure out the config info.
|
||||
* Todo replace this to config module.
|
||||
* Config Helper figure out the config info. Todo replace this to config module.
|
||||
*/
|
||||
public class ConfigHelper {
|
||||
|
||||
|
||||
+1
-2
@@ -21,8 +21,7 @@ package io.ray.streaming.state.config;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* state config keys.
|
||||
* Todo replace this to config module.
|
||||
* state config keys. Todo replace this to config module.
|
||||
*/
|
||||
public final class ConfigKey {
|
||||
|
||||
|
||||
+2
-3
@@ -22,9 +22,8 @@ import com.google.common.base.Preconditions;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* This class defines key-groups. Key-groups is
|
||||
* the key space in a job, which is partitioned for keyed state processing in state backend.
|
||||
* The boundaries of the key-group are inclusive.
|
||||
* This class defines key-groups. Key-groups is the key space in a job, which is partitioned for
|
||||
* keyed state processing in state backend. The boundaries of the key-group are inclusive.
|
||||
*/
|
||||
public class KeyGroup implements Serializable {
|
||||
|
||||
|
||||
+3
-2
@@ -57,8 +57,9 @@ public final class KeyGroupAssignment {
|
||||
return Math.abs(key.hashCode() % maxParallelism);
|
||||
}
|
||||
|
||||
public static Map<Integer, List<Integer>> computeKeyGroupToTask(int maxParallelism,
|
||||
List<Integer> targetTasks) {
|
||||
public static Map<Integer, List<Integer>> computeKeyGroupToTask(
|
||||
int maxParallelism,
|
||||
List<Integer> targetTasks) {
|
||||
Map<Integer, List<Integer>> keyGroupToTask = new ConcurrentHashMap<>();
|
||||
for (int index = 0; index < targetTasks.size(); index++) {
|
||||
KeyGroup taskKeyGroup = getKeyGroup(maxParallelism, targetTasks.size(), index);
|
||||
|
||||
+6
-3
@@ -21,6 +21,7 @@ package io.ray.streaming.state.keystate.desc;
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import io.ray.streaming.state.keystate.state.ListState;
|
||||
|
||||
/**
|
||||
@@ -41,8 +42,9 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
|
||||
return build(name, type, false);
|
||||
}
|
||||
|
||||
public static <T> ListStateDescriptor<T> build(String name, Class<T> type,
|
||||
boolean isOperatorList) {
|
||||
public static <T> ListStateDescriptor<T> build(
|
||||
String name, Class<T> type,
|
||||
boolean isOperatorList) {
|
||||
return new ListStateDescriptor<>(name, type, isOperatorList);
|
||||
}
|
||||
|
||||
@@ -75,7 +77,8 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
|
||||
public String getIdentify() {
|
||||
if (isOperatorList) {
|
||||
return String
|
||||
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, index);
|
||||
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER,
|
||||
index);
|
||||
} else {
|
||||
return super.getIdentify();
|
||||
}
|
||||
|
||||
+3
-2
@@ -31,8 +31,9 @@ public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K
|
||||
// TODO: use the types to help serde
|
||||
}
|
||||
|
||||
public static <K, V> MapStateDescriptor<K, V> build(String name, Class<K> keyType,
|
||||
Class<V> valueType) {
|
||||
public static <K, V> MapStateDescriptor<K, V> build(
|
||||
String name, Class<K> keyType,
|
||||
Class<V> valueType) {
|
||||
return new MapStateDescriptor<>(name, keyType, valueType);
|
||||
}
|
||||
|
||||
|
||||
+3
-3
@@ -20,6 +20,7 @@ package io.ray.streaming.state.keystate.state.impl;
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import io.ray.streaming.state.PartitionRecord;
|
||||
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
|
||||
@@ -30,9 +31,8 @@ import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/**
|
||||
* This class defines the implementation of operator state.
|
||||
* When the state is initialized, we must scan the whole table.
|
||||
* And if the state type is splitList, all the records must be spitted.
|
||||
* This class defines the implementation of operator state. When the state is initialized, we must
|
||||
* scan the whole table. And if the state type is splitList, all the records must be spitted.
|
||||
*/
|
||||
public class OperatorStateImpl<V> implements ListState<V> {
|
||||
|
||||
|
||||
+3
-2
@@ -35,8 +35,9 @@ public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T
|
||||
|
||||
private final ListState<T> listState;
|
||||
|
||||
public ListStateStoreManagerProxy(AbstractKeyStateBackend keyStateBackend,
|
||||
ListStateDescriptor<T> stateDescriptor) {
|
||||
public ListStateStoreManagerProxy(
|
||||
AbstractKeyStateBackend keyStateBackend,
|
||||
ListStateDescriptor<T> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
if (stateDescriptor.isOperatorList()) {
|
||||
this.listState = new OperatorStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
|
||||
+3
-2
@@ -34,8 +34,9 @@ public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<
|
||||
|
||||
private final MapStateImpl<K, V> mapState;
|
||||
|
||||
public MapStateStoreManagerProxy(KeyStateBackend keyStateBackend,
|
||||
MapStateDescriptor<K, V> stateDescriptor) {
|
||||
public MapStateStoreManagerProxy(
|
||||
KeyStateBackend keyStateBackend,
|
||||
MapStateDescriptor<K, V> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
this.mapState = new MapStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
}
|
||||
|
||||
+3
-2
@@ -33,8 +33,9 @@ public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T> im
|
||||
|
||||
private final ValueStateImpl<T> valueState;
|
||||
|
||||
public ValueStateStoreManagerProxy(KeyStateBackend keyStateBackend,
|
||||
ValueStateDescriptor<T> stateDescriptor) {
|
||||
public ValueStateStoreManagerProxy(
|
||||
KeyStateBackend keyStateBackend,
|
||||
ValueStateDescriptor<T> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
this.valueState = new ValueStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,8 +25,8 @@ import org.nustaq.serialization.FSTConfiguration;
|
||||
*/
|
||||
public class Serializer {
|
||||
|
||||
private static final ThreadLocal<FSTConfiguration> conf = ThreadLocal
|
||||
.withInitial(FSTConfiguration::createDefaultConfiguration);
|
||||
private static final ThreadLocal<FSTConfiguration> conf =
|
||||
ThreadLocal.withInitial(FSTConfiguration::createDefaultConfiguration);
|
||||
|
||||
public static byte[] object2Bytes(Object value) {
|
||||
return conf.get().asByteArray(value);
|
||||
|
||||
+2
-2
@@ -28,8 +28,8 @@ import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* This class defines the StoreManager Abstract class.
|
||||
* We use three layer to store the state, frontStore, middleStore and keyValueStore(remote).
|
||||
* This class defines the StoreManager Abstract class. We use three layer to store the state,
|
||||
* frontStore, middleStore and keyValueStore(remote).
|
||||
*/
|
||||
public abstract class AbstractStateStoreManager<V> implements StateStoreManager {
|
||||
|
||||
|
||||
+3
-2
@@ -37,8 +37,9 @@ public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
|
||||
protected final AbstractStateStoreManager<V> stateStrategy;
|
||||
private final AbstractKeyStateBackend keyStateBackend;
|
||||
|
||||
public StateStoreManagerProxy(AbstractKeyStateBackend keyStateBackend,
|
||||
AbstractStateDescriptor stateDescriptor) {
|
||||
public StateStoreManagerProxy(
|
||||
AbstractKeyStateBackend keyStateBackend,
|
||||
AbstractStateDescriptor stateDescriptor) {
|
||||
this.keyStateBackend = keyStateBackend;
|
||||
KeyValueStore<String, Map<Long, byte[]>> backStorage = keyStateBackend
|
||||
.getBackStorage(stateDescriptor);
|
||||
|
||||
+4
-3
@@ -38,7 +38,7 @@ public class KeyStateBackendTest {
|
||||
public void testGetValueState() {
|
||||
keyStateBackend.setCheckpointId(1L);
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("value", String.class, null);
|
||||
.build("value", String.class, null);
|
||||
valueStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
ValueState<String> valueState = keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -120,7 +120,7 @@ public class KeyStateBackendTest {
|
||||
public void testGetListState() {
|
||||
keyStateBackend.setCheckpointId(1l);
|
||||
ListStateDescriptor<String> listStateDescriptor = ListStateDescriptor
|
||||
.build("list", String.class);
|
||||
.build("list", String.class);
|
||||
listStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
ListState<String> listState = keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -208,7 +208,8 @@ public class KeyStateBackendTest {
|
||||
public void testGetMapState() {
|
||||
keyStateBackend.setCheckpointId(1l);
|
||||
MapStateDescriptor<String, String> mapStateDescriptor = MapStateDescriptor
|
||||
.build("map", String.class, String.class);
|
||||
.build("map", String.class,
|
||||
String.class);
|
||||
mapStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
MapState<String, String> mapState = keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
|
||||
+2
-1
@@ -24,7 +24,8 @@ import org.testng.annotations.Test;
|
||||
|
||||
public class DefaultKeyValueStoreSerializationTest {
|
||||
|
||||
DefaultKeyValueStoreSerialization<String, Integer> serDe = new DefaultKeyValueStoreSerialization<>();
|
||||
DefaultKeyValueStoreSerialization<String, Integer> serDe =
|
||||
new DefaultKeyValueStoreSerialization<>();
|
||||
byte[] ret;
|
||||
|
||||
@Test
|
||||
|
||||
+1
@@ -20,6 +20,7 @@ package io.ray.streaming.state.keystate;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class KeyGroupAssignmentTest {
|
||||
|
||||
+2
-1
@@ -26,7 +26,8 @@ public class MapStateDescriptorTest {
|
||||
@Test
|
||||
public void test() {
|
||||
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
|
||||
.build("msdTest", String.class, Integer.class);
|
||||
.build("msdTest", String.class,
|
||||
Integer.class);
|
||||
|
||||
descriptor.setTableName("table");
|
||||
Assert.assertEquals(descriptor.getTableName(), "table");
|
||||
|
||||
+2
-1
@@ -39,7 +39,8 @@ public class MapStateImplTest {
|
||||
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
|
||||
new MemoryStateBackend(new HashMap<>()));
|
||||
MapStateDescriptor<Integer, String> descriptor = MapStateDescriptor
|
||||
.build("MapStateImplTest", Integer.class, String.class);
|
||||
.build("MapStateImplTest", Integer.class,
|
||||
String.class);
|
||||
descriptor.setTableName("table");
|
||||
mapState = (MapStateImpl<Integer, String>) keyStateBackend.getMapState(descriptor);
|
||||
}
|
||||
|
||||
+2
-1
@@ -42,7 +42,8 @@ public class OperatorStateImplTest {
|
||||
operatorStateBackend = new OperatorStateBackend(new MemoryStateBackend(config));
|
||||
|
||||
descriptor = ListStateDescriptor
|
||||
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, true);
|
||||
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class,
|
||||
true);
|
||||
descriptor.setPartitionNumber(1);
|
||||
descriptor.setIndex(0);
|
||||
descriptor.setTableName(table_name);
|
||||
|
||||
+2
-1
@@ -37,7 +37,8 @@ public class ValueStateImplTest {
|
||||
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
|
||||
new MemoryStateBackend(new HashMap<>()));
|
||||
ValueStateDescriptor<String> descriptor = ValueStateDescriptor
|
||||
.build("ValueStateImplTest", String.class, "hello");
|
||||
.build("ValueStateImplTest", String.class,
|
||||
"hello");
|
||||
descriptor.setTableName("table");
|
||||
|
||||
valueState = (ValueStateImpl<String>) keyStateBackend.getValueState(descriptor);
|
||||
|
||||
+8
-4
@@ -55,7 +55,8 @@ public class DualStateStrategyTest {
|
||||
|
||||
public void caseKV() {
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("VALUE-" + currentTime, String.class, defaultValue);
|
||||
.build("VALUE-" + currentTime,
|
||||
String.class, defaultValue);
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -159,7 +160,8 @@ public class DualStateStrategyTest {
|
||||
|
||||
public void caseKVGap() {
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("value2-" + currentTime, String.class, defaultValue);
|
||||
.build("value2-" + currentTime,
|
||||
String.class, defaultValue);
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -208,7 +210,8 @@ public class DualStateStrategyTest {
|
||||
|
||||
public void caseKList() {
|
||||
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
|
||||
.build("LIST-" + currentTime, Integer.class);
|
||||
.build("LIST-" + currentTime,
|
||||
Integer.class);
|
||||
listStateDescriptor.setTableName(table);
|
||||
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -310,7 +313,8 @@ public class DualStateStrategyTest {
|
||||
|
||||
public void caseKMap() {
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
|
||||
.build("MAP-" + currentTime, Integer.class, Integer.class);
|
||||
.build("MAP-" + currentTime,
|
||||
Integer.class, Integer.class);
|
||||
mapStateDescriptor.setTableName(table);
|
||||
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
|
||||
+6
-3
@@ -53,7 +53,8 @@ public class MVStateStrategyTest {
|
||||
public void caseKV() {
|
||||
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("mvint-" + currentTime, String.class, "");
|
||||
.build("mvint-" + currentTime,
|
||||
String.class, "");
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -170,7 +171,8 @@ public class MVStateStrategyTest {
|
||||
|
||||
public void caseKList() {
|
||||
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
|
||||
.build("mvlist-" + currentTime, Integer.class);
|
||||
.build("mvlist-" + currentTime,
|
||||
Integer.class);
|
||||
listStateDescriptor.setTableName(table);
|
||||
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -274,7 +276,8 @@ public class MVStateStrategyTest {
|
||||
|
||||
public void caseKMap() {
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
|
||||
.build("mvmap-" + currentTime, Integer.class, Integer.class);
|
||||
.build("mvmap-" + currentTime,
|
||||
Integer.class, Integer.class);
|
||||
mapStateDescriptor.setTableName(table);
|
||||
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user