[Streaming]Format java code using IDEA (#10440)

This commit is contained in:
Lixin Wei
2020-08-31 19:45:00 +08:00
committed by GitHub
parent afde3db4f0
commit 6bde6b493e
125 changed files with 548 additions and 433 deletions
@@ -21,21 +21,20 @@ package io.ray.streaming.state;
/**
* TransactionState interface.
* <p>
* Streaming State should implement transaction in case of failure,
* which in our case is four default method, finish, commit, ackCommit, rollback.
* Streaming State should implement transaction in case of failure, which in our case is four
* default method, finish, commit, ackCommit, rollback.
*/
public interface StateStoreManager {
/**
* The finish method is used when the batched data is all saved in state.
* Normally, serialization job is done here.
* The finish method is used when the batched data is all saved in state. Normally, serialization
* job is done here.
*/
void finish(long checkpointId);
/**
* The commit method is used for persistent, and can be used in another thread to reach async
* state commit.
* Normally, data persistent is done here.
* state commit. Normally, data persistent is done here.
*/
void commit(long checkpointId);
@@ -20,6 +20,7 @@ package io.ray.streaming.state.backend;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import io.ray.streaming.state.config.ConfigKey;
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
@@ -31,16 +31,17 @@ import io.ray.streaming.state.keystate.state.proxy.MapStateStoreManagerProxy;
import io.ray.streaming.state.keystate.state.proxy.ValueStateStoreManagerProxy;
/**
* key state backend manager, managing different kinds of states in different thread.
* This class is not thread safe.
* key state backend manager, managing different kinds of states in different thread. This class is
* not thread safe.
*/
public class KeyStateBackend extends AbstractKeyStateBackend {
protected final int numberOfKeyGroups;
protected final KeyGroup keyGroup;
public KeyStateBackend(int numberOfKeyGroups, KeyGroup keyGroup,
AbstractStateBackend abstractStateBackend) {
public KeyStateBackend(
int numberOfKeyGroups, KeyGroup keyGroup,
AbstractStateBackend abstractStateBackend) {
super(abstractStateBackend);
this.numberOfKeyGroups = numberOfKeyGroups;
this.keyGroup = keyGroup;
@@ -27,8 +27,9 @@ import java.util.Map;
*/
public class StateBackendBuilder {
private static AbstractStateBackend getStateBackend(Map<String, String> config,
BackendType type) {
private static AbstractStateBackend getStateBackend(
Map<String, String> config,
BackendType type) {
switch (type) {
case MEMORY:
return new MemoryStateBackend(config);
@@ -21,8 +21,7 @@ package io.ray.streaming.state.config;
import java.util.Map;
/**
* Config Helper figure out the config info.
* Todo replace this to config module.
* Config Helper figure out the config info. Todo replace this to config module.
*/
public class ConfigHelper {
@@ -21,8 +21,7 @@ package io.ray.streaming.state.config;
import java.util.Map;
/**
* state config keys.
* Todo replace this to config module.
* state config keys. Todo replace this to config module.
*/
public final class ConfigKey {
@@ -22,9 +22,8 @@ import com.google.common.base.Preconditions;
import java.io.Serializable;
/**
* This class defines key-groups. Key-groups is
* the key space in a job, which is partitioned for keyed state processing in state backend.
* The boundaries of the key-group are inclusive.
* This class defines key-groups. Key-groups is the key space in a job, which is partitioned for
* keyed state processing in state backend. The boundaries of the key-group are inclusive.
*/
public class KeyGroup implements Serializable {
@@ -57,8 +57,9 @@ public final class KeyGroupAssignment {
return Math.abs(key.hashCode() % maxParallelism);
}
public static Map<Integer, List<Integer>> computeKeyGroupToTask(int maxParallelism,
List<Integer> targetTasks) {
public static Map<Integer, List<Integer>> computeKeyGroupToTask(
int maxParallelism,
List<Integer> targetTasks) {
Map<Integer, List<Integer>> keyGroupToTask = new ConcurrentHashMap<>();
for (int index = 0; index < targetTasks.size(); index++) {
KeyGroup taskKeyGroup = getKeyGroup(maxParallelism, targetTasks.size(), index);
@@ -21,6 +21,7 @@ package io.ray.streaming.state.keystate.desc;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import io.ray.streaming.state.keystate.state.ListState;
/**
@@ -41,8 +42,9 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
return build(name, type, false);
}
public static <T> ListStateDescriptor<T> build(String name, Class<T> type,
boolean isOperatorList) {
public static <T> ListStateDescriptor<T> build(
String name, Class<T> type,
boolean isOperatorList) {
return new ListStateDescriptor<>(name, type, isOperatorList);
}
@@ -75,7 +77,8 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
public String getIdentify() {
if (isOperatorList) {
return String
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, index);
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER,
index);
} else {
return super.getIdentify();
}
@@ -31,8 +31,9 @@ public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K
// TODO: use the types to help serde
}
public static <K, V> MapStateDescriptor<K, V> build(String name, Class<K> keyType,
Class<V> valueType) {
public static <K, V> MapStateDescriptor<K, V> build(
String name, Class<K> keyType,
Class<V> valueType) {
return new MapStateDescriptor<>(name, keyType, valueType);
}
@@ -20,6 +20,7 @@ package io.ray.streaming.state.keystate.state.impl;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import com.google.common.base.Preconditions;
import io.ray.streaming.state.PartitionRecord;
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
@@ -30,9 +31,8 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* This class defines the implementation of operator state.
* When the state is initialized, we must scan the whole table.
* And if the state type is splitList, all the records must be spitted.
* This class defines the implementation of operator state. When the state is initialized, we must
* scan the whole table. And if the state type is splitList, all the records must be spitted.
*/
public class OperatorStateImpl<V> implements ListState<V> {
@@ -35,8 +35,9 @@ public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T
private final ListState<T> listState;
public ListStateStoreManagerProxy(AbstractKeyStateBackend keyStateBackend,
ListStateDescriptor<T> stateDescriptor) {
public ListStateStoreManagerProxy(
AbstractKeyStateBackend keyStateBackend,
ListStateDescriptor<T> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
if (stateDescriptor.isOperatorList()) {
this.listState = new OperatorStateImpl<>(stateDescriptor, keyStateBackend);
@@ -34,8 +34,9 @@ public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<
private final MapStateImpl<K, V> mapState;
public MapStateStoreManagerProxy(KeyStateBackend keyStateBackend,
MapStateDescriptor<K, V> stateDescriptor) {
public MapStateStoreManagerProxy(
KeyStateBackend keyStateBackend,
MapStateDescriptor<K, V> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
this.mapState = new MapStateImpl<>(stateDescriptor, keyStateBackend);
}
@@ -33,8 +33,9 @@ public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T> im
private final ValueStateImpl<T> valueState;
public ValueStateStoreManagerProxy(KeyStateBackend keyStateBackend,
ValueStateDescriptor<T> stateDescriptor) {
public ValueStateStoreManagerProxy(
KeyStateBackend keyStateBackend,
ValueStateDescriptor<T> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
this.valueState = new ValueStateImpl<>(stateDescriptor, keyStateBackend);
}
@@ -25,8 +25,8 @@ import org.nustaq.serialization.FSTConfiguration;
*/
public class Serializer {
private static final ThreadLocal<FSTConfiguration> conf = ThreadLocal
.withInitial(FSTConfiguration::createDefaultConfiguration);
private static final ThreadLocal<FSTConfiguration> conf =
ThreadLocal.withInitial(FSTConfiguration::createDefaultConfiguration);
public static byte[] object2Bytes(Object value) {
return conf.get().asByteArray(value);
@@ -28,8 +28,8 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* This class defines the StoreManager Abstract class.
* We use three layer to store the state, frontStore, middleStore and keyValueStore(remote).
* This class defines the StoreManager Abstract class. We use three layer to store the state,
* frontStore, middleStore and keyValueStore(remote).
*/
public abstract class AbstractStateStoreManager<V> implements StateStoreManager {
@@ -37,8 +37,9 @@ public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
protected final AbstractStateStoreManager<V> stateStrategy;
private final AbstractKeyStateBackend keyStateBackend;
public StateStoreManagerProxy(AbstractKeyStateBackend keyStateBackend,
AbstractStateDescriptor stateDescriptor) {
public StateStoreManagerProxy(
AbstractKeyStateBackend keyStateBackend,
AbstractStateDescriptor stateDescriptor) {
this.keyStateBackend = keyStateBackend;
KeyValueStore<String, Map<Long, byte[]>> backStorage = keyStateBackend
.getBackStorage(stateDescriptor);
@@ -38,7 +38,7 @@ public class KeyStateBackendTest {
public void testGetValueState() {
keyStateBackend.setCheckpointId(1L);
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("value", String.class, null);
.build("value", String.class, null);
valueStateDescriptor.setTableName("kepler_hlg_ut");
ValueState<String> valueState = keyStateBackend.getValueState(valueStateDescriptor);
@@ -120,7 +120,7 @@ public class KeyStateBackendTest {
public void testGetListState() {
keyStateBackend.setCheckpointId(1l);
ListStateDescriptor<String> listStateDescriptor = ListStateDescriptor
.build("list", String.class);
.build("list", String.class);
listStateDescriptor.setTableName("kepler_hlg_ut");
ListState<String> listState = keyStateBackend.getListState(listStateDescriptor);
@@ -208,7 +208,8 @@ public class KeyStateBackendTest {
public void testGetMapState() {
keyStateBackend.setCheckpointId(1l);
MapStateDescriptor<String, String> mapStateDescriptor = MapStateDescriptor
.build("map", String.class, String.class);
.build("map", String.class,
String.class);
mapStateDescriptor.setTableName("kepler_hlg_ut");
MapState<String, String> mapState = keyStateBackend.getMapState(mapStateDescriptor);
@@ -24,7 +24,8 @@ import org.testng.annotations.Test;
public class DefaultKeyValueStoreSerializationTest {
DefaultKeyValueStoreSerialization<String, Integer> serDe = new DefaultKeyValueStoreSerialization<>();
DefaultKeyValueStoreSerialization<String, Integer> serDe =
new DefaultKeyValueStoreSerialization<>();
byte[] ret;
@Test
@@ -20,6 +20,7 @@ package io.ray.streaming.state.keystate;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.Test;
public class KeyGroupAssignmentTest {
@@ -26,7 +26,8 @@ public class MapStateDescriptorTest {
@Test
public void test() {
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
.build("msdTest", String.class, Integer.class);
.build("msdTest", String.class,
Integer.class);
descriptor.setTableName("table");
Assert.assertEquals(descriptor.getTableName(), "table");
@@ -39,7 +39,8 @@ public class MapStateImplTest {
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
new MemoryStateBackend(new HashMap<>()));
MapStateDescriptor<Integer, String> descriptor = MapStateDescriptor
.build("MapStateImplTest", Integer.class, String.class);
.build("MapStateImplTest", Integer.class,
String.class);
descriptor.setTableName("table");
mapState = (MapStateImpl<Integer, String>) keyStateBackend.getMapState(descriptor);
}
@@ -42,7 +42,8 @@ public class OperatorStateImplTest {
operatorStateBackend = new OperatorStateBackend(new MemoryStateBackend(config));
descriptor = ListStateDescriptor
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, true);
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class,
true);
descriptor.setPartitionNumber(1);
descriptor.setIndex(0);
descriptor.setTableName(table_name);
@@ -37,7 +37,8 @@ public class ValueStateImplTest {
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
new MemoryStateBackend(new HashMap<>()));
ValueStateDescriptor<String> descriptor = ValueStateDescriptor
.build("ValueStateImplTest", String.class, "hello");
.build("ValueStateImplTest", String.class,
"hello");
descriptor.setTableName("table");
valueState = (ValueStateImpl<String>) keyStateBackend.getValueState(descriptor);
@@ -55,7 +55,8 @@ public class DualStateStrategyTest {
public void caseKV() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("VALUE-" + currentTime, String.class, defaultValue);
.build("VALUE-" + currentTime,
String.class, defaultValue);
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -159,7 +160,8 @@ public class DualStateStrategyTest {
public void caseKVGap() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("value2-" + currentTime, String.class, defaultValue);
.build("value2-" + currentTime,
String.class, defaultValue);
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -208,7 +210,8 @@ public class DualStateStrategyTest {
public void caseKList() {
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
.build("LIST-" + currentTime, Integer.class);
.build("LIST-" + currentTime,
Integer.class);
listStateDescriptor.setTableName(table);
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
@@ -310,7 +313,8 @@ public class DualStateStrategyTest {
public void caseKMap() {
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
.build("MAP-" + currentTime, Integer.class, Integer.class);
.build("MAP-" + currentTime,
Integer.class, Integer.class);
mapStateDescriptor.setTableName(table);
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
@@ -53,7 +53,8 @@ public class MVStateStrategyTest {
public void caseKV() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("mvint-" + currentTime, String.class, "");
.build("mvint-" + currentTime,
String.class, "");
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -170,7 +171,8 @@ public class MVStateStrategyTest {
public void caseKList() {
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
.build("mvlist-" + currentTime, Integer.class);
.build("mvlist-" + currentTime,
Integer.class);
listStateDescriptor.setTableName(table);
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
@@ -274,7 +276,8 @@ public class MVStateStrategyTest {
public void caseKMap() {
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
.build("mvmap-" + currentTime, Integer.class, Integer.class);
.build("mvmap-" + currentTime,
Integer.class, Integer.class);
mapStateDescriptor.setTableName(table);
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);