[Java] Format ray java code (#13056)

This commit is contained in:
chaokunyang
2020-12-29 10:36:16 +08:00
committed by GitHub
parent cc1c2c3dc9
commit d1dd3410c8
422 changed files with 4384 additions and 5035 deletions
@@ -18,18 +18,12 @@
package io.ray.streaming.state;
/**
* Key Value State interface.
*/
/** Key Value State interface. */
public interface KeyValueState<K, V> {
/**
* get value from state
*/
/** get value from state */
V get(K key);
/**
* put key and value into state
*/
/** put key and value into state */
void put(K k, V v);
}
@@ -20,19 +20,15 @@ package io.ray.streaming.state;
import java.io.Serializable;
/**
* value record for partition.
*/
/** value record for partition. */
public class PartitionRecord<T> implements Serializable {
/**
* The partition number of the partitioned value.
*/
/** The partition number of the partitioned value. */
private int partitionID;
private T value;
public PartitionRecord() {
}
public PartitionRecord() {}
public PartitionRecord(int partitionID, T value) {
this.partitionID = partitionID;
@@ -18,9 +18,7 @@
package io.ray.streaming.state;
/**
* RuntimeException wrapper, indicating the exceptions occurs in states.
*/
/** RuntimeException wrapper, indicating the exceptions occurs in states. */
public class StateException extends RuntimeException {
public StateException(Throwable t) {
@@ -20,8 +20,8 @@ package io.ray.streaming.state;
/**
* TransactionState interface.
* <p>
* Streaming State should implement transaction in case of failure, which in our case is four
*
* <p>Streaming State should implement transaction in case of failure, which in our case is four
* default method, finish, commit, ackCommit, rollback.
*/
public interface StateStoreManager {
@@ -44,8 +44,6 @@ public interface StateStoreManager {
*/
void ackCommit(long checkpointId, long timeStamp);
/**
* The rollback method is used for recovering the checkpoint.
*/
/** The rollback method is used for recovering the checkpoint. */
void rollBack(long checkpointId);
}
@@ -20,16 +20,13 @@ package io.ray.streaming.state;
import java.io.Serializable;
/**
* This Class contains a record with some checkpointId.
*/
/** This Class contains a record with some checkpointId. */
public class StorageRecord<T> implements Serializable {
private long checkpointId;
private T value;
public StorageRecord() {
}
public StorageRecord() {}
public StorageRecord(long checkpointId, T value) {
this.checkpointId = checkpointId;
@@ -36,8 +36,8 @@ import org.slf4j.LoggerFactory;
/**
* Transaction support primitive operations like finish, commit, ackcommit and rollback.
* <p>
* State value modification is not thread safe! By default, every processing thread has its own
*
* <p>State value modification is not thread safe! By default, every processing thread has its own
* space to handle state.
*/
public abstract class AbstractKeyStateBackend implements StateStoreManager {
@@ -52,10 +52,9 @@ public abstract class AbstractKeyStateBackend implements StateStoreManager {
protected Map<String, MapStateStoreManagerProxy> mapManagerProxyHashMap = new HashMap<>();
protected Set<String> descNamespace;
/**
* tablename, KeyValueStore key, checkpointId, content
*/
/** tablename, KeyValueStore key, checkpointId, content */
protected Map<String, KeyValueStore<String, Map<Long, byte[]>>> backStorageCache;
private AbstractStateBackend backend;
public AbstractKeyStateBackend(AbstractStateBackend backend) {
@@ -20,7 +20,6 @@ package io.ray.streaming.state.backend;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import io.ray.streaming.state.config.ConfigKey;
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
@@ -29,9 +28,7 @@ import io.ray.streaming.state.store.KeyValueStore;
import java.io.Serializable;
import java.util.Map;
/**
* This class is the abstract class for different kinds of StateBackend.
*/
/** This class is the abstract class for different kinds of StateBackend. */
public abstract class AbstractStateBackend implements Serializable {
protected final Map<String, String> config;
@@ -18,18 +18,12 @@
package io.ray.streaming.state.backend;
/**
* Backend Types.
*/
/** Backend Types. */
public enum BackendType {
/**
* Saving the state values in memory
*/
/** Saving the state values in memory */
MEMORY;
/**
* get the enum from input string value, ignoring the case
*/
/** get the enum from input string value, ignoring the case */
public static BackendType getEnum(String value) {
for (BackendType v : values()) {
if (v.name().equalsIgnoreCase(value)) {
@@ -40,16 +40,13 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
protected final KeyGroup keyGroup;
public KeyStateBackend(
int numberOfKeyGroups, KeyGroup keyGroup,
AbstractStateBackend abstractStateBackend) {
int numberOfKeyGroups, KeyGroup keyGroup, AbstractStateBackend abstractStateBackend) {
super(abstractStateBackend);
this.numberOfKeyGroups = numberOfKeyGroups;
this.keyGroup = keyGroup;
}
/**
* get new value state proxy
*/
/** get new value state proxy */
protected <T> ValueStateStoreManagerProxy<T> newValueStateProxy(
ValueStateDescriptor<T> stateDescriptor) {
return new ValueStateStoreManagerProxy<>(this, stateDescriptor);
@@ -66,9 +63,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
}
}
/**
* get new list state proxy
*/
/** get new list state proxy */
protected <T> ListStateStoreManagerProxy<T> newListStateProxy(
ListStateDescriptor<T> stateDescriptor) {
return new ListStateStoreManagerProxy<>(this, stateDescriptor);
@@ -86,9 +81,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
}
}
/**
* get map state proxy
*/
/** get map state proxy */
protected <S, T> MapStateStoreManagerProxy<S, T> newMapStateProxy(
MapStateDescriptor<S, T> stateDescriptor) {
return new MapStateStoreManagerProxy<>(this, stateDescriptor);
@@ -108,8 +101,8 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
@Override
public void setCurrentKey(Object currentKey) {
super.keyGroupIndex = KeyGroupAssignment
.assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups);
super.keyGroupIndex =
KeyGroupAssignment.assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups);
super.currentKey = currentKey;
}
@@ -23,9 +23,7 @@ import io.ray.streaming.state.keystate.state.ListState;
import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl;
import io.ray.streaming.state.keystate.state.proxy.ListStateStoreManagerProxy;
/**
* OperatorState manager for getting split or union list state.
*/
/** OperatorState manager for getting split or union list state. */
public class OperatorStateBackend extends AbstractKeyStateBackend {
public OperatorStateBackend(AbstractStateBackend backend) {
@@ -42,9 +40,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend {
return new ListStateStoreManagerProxy<>(this, stateDescriptor);
}
/**
* get spitted List for different operator instance.
*/
/** get spitted List for different operator instance. */
public <T> ListState<T> getSplitListState(ListStateDescriptor<T> stateDescriptor) {
String desc = stateDescriptor.getIdentify();
if (listManagerProxyHashMap.containsKey(desc)) {
@@ -58,9 +54,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend {
}
}
/**
* get a union List for different operator instance.
*/
/** get a union List for different operator instance. */
public <T> ListState<T> getUnionListState(ListStateDescriptor<T> stateDescriptor) {
String desc = stateDescriptor.getIdentify();
if (listManagerProxyHashMap.containsKey(desc)) {
@@ -22,14 +22,11 @@ import io.ray.streaming.state.backend.impl.MemoryStateBackend;
import io.ray.streaming.state.config.ConfigKey;
import java.util.Map;
/**
* State Backend Builder.
*/
/** State Backend Builder. */
public class StateBackendBuilder {
private static AbstractStateBackend getStateBackend(
Map<String, String> config,
BackendType type) {
Map<String, String> config, BackendType type) {
switch (type) {
case MEMORY:
return new MemoryStateBackend(config);
@@ -18,21 +18,14 @@
package io.ray.streaming.state.backend;
/**
* This class describe State Saving Model.
*/
/** This class describe State Saving Model. */
public enum StateStrategy {
/**
* save two version together in case of rollback.
*/
/** save two version together in case of rollback. */
DUAL_VERSION,
/**
* for storage supporting mvcc, we save only current version.
*/
/** for storage supporting mvcc, we save only current version. */
SINGLE_VERSION;
public static StateStrategy getEnum(String value) {
for (StateStrategy v : values()) {
if (v.name().equalsIgnoreCase(value)) {
@@ -26,9 +26,7 @@ import io.ray.streaming.state.store.impl.MemoryKeyMapStore;
import io.ray.streaming.state.store.impl.MemoryKeyValueStore;
import java.util.Map;
/**
* MemoryStateBackend. Supporting memory store.
*/
/** MemoryStateBackend. Supporting memory store. */
public class MemoryStateBackend extends AbstractStateBackend {
public MemoryStateBackend(Map<String, String> config) {
@@ -50,5 +48,4 @@ public class MemoryStateBackend extends AbstractStateBackend {
String tableName, KeyMapStoreSerializer keyMapStoreSerializer) {
return new MemoryKeyMapStore<>();
}
}
@@ -20,9 +20,7 @@ package io.ray.streaming.state.config;
import java.util.Map;
/**
* Config Helper figure out the config info. Todo replace this to config module.
*/
/** Config Helper figure out the config info. Todo replace this to config module. */
public class ConfigHelper {
public static int getIntegerOrDefault(Map config, String configKey, int defaultValue) {
@@ -40,5 +38,4 @@ public class ConfigHelper {
return defaultValue;
}
}
}
@@ -20,15 +20,12 @@ package io.ray.streaming.state.config;
import java.util.Map;
/**
* state config keys. Todo replace this to config module.
*/
/** state config keys. Todo replace this to config module. */
public final class ConfigKey {
/**
* backend
*/
/** backend */
public static final String STATE_BACKEND_TYPE = "state.backend.type";
public static final String STATE_TABLE_NAME = "state.table.name";
public static final String STATE_STRATEGY_MODE = "state.strategy.mode";
public static final String NUMBER_PER_CHECKPOINT = "number.per.checkpoint";
@@ -43,9 +43,7 @@ public class KeyGroup implements Serializable {
Preconditions.checkArgument(size() >= 0, "overflow detected.");
}
/**
* @return The number of key-group in the range
*/
/** Returns The number of key-group in the range */
public int size() {
return 1 + endIndex - startIndex;
}
@@ -24,9 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* This class defines key-group assignment algorithm。
*/
/** This class defines key-group assignment algorithm。 */
public final class KeyGroupAssignment {
/**
@@ -37,8 +35,10 @@ public final class KeyGroupAssignment {
* @param index index of the operator instance.
*/
public static KeyGroup getKeyGroup(int maxParallelism, int parallelism, int index) {
Preconditions.checkArgument(maxParallelism >= parallelism,
"Maximum parallelism (%s) must not be smaller than parallelism(%s)", maxParallelism,
Preconditions.checkArgument(
maxParallelism >= parallelism,
"Maximum parallelism (%s) must not be smaller than parallelism(%s)",
maxParallelism,
parallelism);
int start = index == 0 ? 0 : ((index * maxParallelism - 1) / parallelism) + 1;
@@ -50,25 +50,24 @@ public final class KeyGroupAssignment {
* Assigning the key to a key-group index.
*
* @param key the key to assign.
* @param maxParallelism the maximum parallelism.
* @return the key-group index to which the given key is assigned.
* @param maxParallelism the maximum parallelism. Returns the key-group index to which the given
* key is assigned.
*/
public static int assignKeyGroupIndexForKey(Object key, int maxParallelism) {
return Math.abs(key.hashCode() % maxParallelism);
}
public static Map<Integer, List<Integer>> computeKeyGroupToTask(
int maxParallelism,
List<Integer> targetTasks) {
int maxParallelism, List<Integer> targetTasks) {
Map<Integer, List<Integer>> keyGroupToTask = new ConcurrentHashMap<>();
for (int index = 0; index < targetTasks.size(); index++) {
KeyGroup taskKeyGroup = getKeyGroup(maxParallelism, targetTasks.size(), index);
for (int groupId = taskKeyGroup.getStartIndex(); groupId <= taskKeyGroup.getEndIndex();
for (int groupId = taskKeyGroup.getStartIndex();
groupId <= taskKeyGroup.getEndIndex();
groupId++) {
keyGroupToTask.put(groupId, ImmutableList.of(targetTasks.get(index)));
}
}
return keyGroupToTask;
}
}
@@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc;
import com.google.common.base.Preconditions;
import io.ray.streaming.state.keystate.state.State;
/**
* This class defines basic data structures of StateDescriptor.
*/
/** This class defines basic data structures of StateDescriptor. */
public abstract class AbstractStateDescriptor<S extends State, T> {
private final String name;
@@ -65,24 +63,26 @@ public abstract class AbstractStateDescriptor<S extends State, T> {
@Override
public String toString() {
return "AbstractStateDescriptor{" + "tableName='" + tableName + '\'' + ", name='" + name + '\''
+ ", type=" + type + '}';
return "AbstractStateDescriptor{"
+ "tableName='"
+ tableName
+ '\''
+ ", name='"
+ name
+ '\''
+ ", type="
+ type
+ '}';
}
public enum StateType {
/**
* value state
*/
/** value state */
VALUE,
/**
* list state
*/
/** list state */
LIST,
/**
* map state
*/
/** map state */
MAP
}
}
@@ -18,15 +18,11 @@
package io.ray.streaming.state.keystate.desc;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import io.ray.streaming.state.keystate.state.ListState;
/**
* ListStateDescriptor.
*/
/** ListStateDescriptor. */
public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>, T> {
private final boolean isOperatorList;
@@ -43,8 +39,7 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
}
public static <T> ListStateDescriptor<T> build(
String name, Class<T> type,
boolean isOperatorList) {
String name, Class<T> type, boolean isOperatorList) {
return new ListStateDescriptor<>(name, type, isOperatorList);
}
@@ -76,9 +71,8 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
@Override
public String getIdentify() {
if (isOperatorList) {
return String
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER,
index);
return String.format(
"%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, index);
} else {
return super.getIdentify();
}
@@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc;
import io.ray.streaming.state.keystate.state.MapState;
import java.util.Map;
/**
* MapStateDescriptor.
*/
/** MapStateDescriptor. */
public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K, V>, Map<K, V>> {
public MapStateDescriptor(String name, Class<K> keyType, Class<V> valueType) {
@@ -32,8 +30,7 @@ public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K
}
public static <K, V> MapStateDescriptor<K, V> build(
String name, Class<K> keyType,
Class<V> valueType) {
String name, Class<K> keyType, Class<V> valueType) {
return new MapStateDescriptor<>(name, keyType, valueType);
}
@@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.desc;
import io.ray.streaming.state.keystate.state.ValueState;
/**
* ValueStateDescriptor.
*/
/** ValueStateDescriptor. */
public class ValueStateDescriptor<T> extends AbstractStateDescriptor<ValueState<T>, T> {
private final T defaultValue;
@@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.state;
import java.util.List;
/**
* ListState interface.
*/
/** ListState interface. */
public interface ListState<T> extends UnaryState<List<T>> {
/**
@@ -22,16 +22,13 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
/**
* MapState interface.
*/
/** MapState interface. */
public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Returns the current value associated with the given key.
*
* @param key The key of the mapping
* @return The value of the mapping with the given key
* @param key The key of the mapping Returns The value of the mapping with the given key
*/
V get(K key);
@@ -67,8 +64,8 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Returns whether there exists the given mapping.
*
* @param key The key of the mapping
* @return True if there exists a mapping whose key equals to the given key
* @param key The key of the mapping Returns True if there exists a mapping whose key equals to
* the given key
*/
default boolean contains(K key) {
return get().containsKey(key);
@@ -77,7 +74,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Returns all the mappings in the state
*
* @return An iterable view of all the key-value pairs in the state.
* <p>Returns An iterable view of all the key-value pairs in the state.
*/
default Iterable<Entry<K, V>> entries() {
return get().entrySet();
@@ -86,7 +83,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Returns all the keys in the state
*
* @return An iterable view of all the keys in the state.
* <p>Returns An iterable view of all the keys in the state.
*/
default Iterable<K> keys() {
return get().keySet();
@@ -95,7 +92,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Returns all the values in the state.
*
* @return An iterable view of all the values in the state.
* <p>Returns An iterable view of all the values in the state.
*/
default Iterable<V> values() {
return get().values();
@@ -104,7 +101,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
/**
* Iterates over all the mappings in the state.
*
* @return An iterator over all the mappings in the state
* <p>Returns An iterator over all the mappings in the state
*/
default Iterator<Entry<K, V>> iterator() {
return get().entrySet().iterator();
@@ -18,13 +18,9 @@
package io.ray.streaming.state.keystate.state;
/**
* State interface.
*/
/** State interface. */
public interface State {
/**
* set current key of the state
*/
/** set current key of the state */
void setCurrentKey(Object currentKey);
}
@@ -18,15 +18,13 @@
package io.ray.streaming.state.keystate.state;
/**
* one value per state.
*/
/** one value per state. */
public interface UnaryState<O> extends State {
/**
* get the value in state
*
* @return the value in state
* <p>Returns the value in state
*/
O get();
}
}
@@ -18,9 +18,7 @@
package io.ray.streaming.state.keystate.state;
/**
* ValueState interface.
*/
/** ValueState interface. */
public interface ValueState<T> extends UnaryState<T> {
/**
@@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.ListState;
import java.util.ArrayList;
import java.util.List;
/**
* ListState implementation.
*/
/** ListState implementation. */
public class ListStateImpl<V> implements ListState<V> {
private final StateHelper<List<V>> helper;
@@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.MapState;
import java.util.HashMap;
import java.util.Map;
/**
* MapState implementation.
*/
/** MapState implementation. */
public class MapStateImpl<K, V> implements MapState<K, V> {
private final StateHelper<Map<K, V>> helper;
@@ -82,9 +80,7 @@ public class MapStateImpl<K, V> implements MapState<K, V> {
helper.put(map);
}
/**
* set current key of the state
*/
/** set current key of the state */
@Override
public void setCurrentKey(Object currentKey) {
helper.setCurrentKey(currentKey);
@@ -20,7 +20,6 @@ package io.ray.streaming.state.keystate.state.impl;
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
import com.google.common.base.Preconditions;
import io.ray.streaming.state.PartitionRecord;
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
@@ -73,8 +72,8 @@ public class OperatorStateImpl<V> implements ListState<V> {
int partitionNum = -1;
int index = 0;
while (true) {
List<PartitionRecord<V>> list = helper.getBackend()
.get(descriptor, getKey(descriptor.getIdentify(), index));
List<PartitionRecord<V>> list =
helper.getBackend().get(descriptor, getKey(descriptor.getIdentify(), index));
if (list != null && !list.isEmpty()) {
partitionNum = list.get(0).getPartitionID();
allList.addAll(list);
@@ -22,9 +22,7 @@ import com.google.common.base.Preconditions;
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
/**
* State Helper Class.
*/
/** State Helper Class. */
public class StateHelper<T> {
private final AbstractKeyStateBackend backend;
@@ -22,9 +22,7 @@ import io.ray.streaming.state.backend.KeyStateBackend;
import io.ray.streaming.state.keystate.desc.ValueStateDescriptor;
import io.ray.streaming.state.keystate.state.ValueState;
/**
* ValueState implementation.
*/
/** ValueState implementation. */
public class ValueStateImpl<T> implements ValueState<T> {
private final StateHelper<T> helper;
@@ -48,9 +46,7 @@ public class ValueStateImpl<T> implements ValueState<T> {
}
}
/**
* set current key of the state
*/
/** set current key of the state */
@Override
public void setCurrentKey(Object currentKey) {
helper.setCurrentKey(currentKey);
@@ -27,17 +27,14 @@ import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl;
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
import java.util.List;
/**
* This class defines ListState Wrapper, connecting state and backend.
*/
public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T>> implements
KeyValueState<String, List<T>> {
/** This class defines ListState Wrapper, connecting state and backend. */
public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T>>
implements KeyValueState<String, List<T>> {
private final ListState<T> listState;
public ListStateStoreManagerProxy(
AbstractKeyStateBackend keyStateBackend,
ListStateDescriptor<T> stateDescriptor) {
AbstractKeyStateBackend keyStateBackend, ListStateDescriptor<T> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
if (stateDescriptor.isOperatorList()) {
this.listState = new OperatorStateImpl<>(stateDescriptor, keyStateBackend);
@@ -26,17 +26,14 @@ import io.ray.streaming.state.keystate.state.impl.MapStateImpl;
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
import java.util.Map;
/**
* This class defines MapState Wrapper, connecting state and backend.
*/
public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<K, V>> implements
KeyValueState<String, Map<K, V>> {
/** This class defines MapState Wrapper, connecting state and backend. */
public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<K, V>>
implements KeyValueState<String, Map<K, V>> {
private final MapStateImpl<K, V> mapState;
public MapStateStoreManagerProxy(
KeyStateBackend keyStateBackend,
MapStateDescriptor<K, V> stateDescriptor) {
KeyStateBackend keyStateBackend, MapStateDescriptor<K, V> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
this.mapState = new MapStateImpl<>(stateDescriptor, keyStateBackend);
}
@@ -25,17 +25,14 @@ import io.ray.streaming.state.keystate.state.ValueState;
import io.ray.streaming.state.keystate.state.impl.ValueStateImpl;
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
/**
* This class defines ValueState Wrapper, connecting state and backend.
*/
public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T> implements
KeyValueState<String, T> {
/** This class defines ValueState Wrapper, connecting state and backend. */
public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T>
implements KeyValueState<String, T> {
private final ValueStateImpl<T> valueState;
public ValueStateStoreManagerProxy(
KeyStateBackend keyStateBackend,
ValueStateDescriptor<T> stateDescriptor) {
KeyStateBackend keyStateBackend, ValueStateDescriptor<T> stateDescriptor) {
super(keyStateBackend, stateDescriptor);
this.valueState = new ValueStateImpl<>(stateDescriptor, keyStateBackend);
}
@@ -18,9 +18,7 @@
package io.ray.streaming.state.serialization;
/**
* Key Map Serialization and Deserialization.
*/
/** Key Map Serialization and Deserialization. */
public interface KeyMapStoreSerializer<K, S, T> {
byte[] serializeKey(K key);
@@ -32,5 +30,4 @@ public interface KeyMapStoreSerializer<K, S, T> {
byte[] serializeUValue(T uv);
T deserializeUValue(byte[] uvArray);
}
@@ -18,9 +18,7 @@
package io.ray.streaming.state.serialization;
/**
* Key Value Serialization and Deserialization.
*/
/** Key Value Serialization and Deserialization. */
public interface KeyValueStoreSerialization<K, V> {
byte[] serializeKey(K key);
@@ -20,9 +20,7 @@ package io.ray.streaming.state.serialization;
import org.nustaq.serialization.FSTConfiguration;
/**
* fst wrapper.
*/
/** fst wrapper. */
public class Serializer {
private static final ThreadLocal<FSTConfiguration> conf =
@@ -24,9 +24,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* AbstractSerialization. Generate row key.
*/
/** AbstractSerialization. Generate row key. */
public abstract class AbstractSerialization {
private static final Logger LOG = LoggerFactory.getLogger(AbstractSerialization.class);
@@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl;
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
import io.ray.streaming.state.serialization.Serializer;
/**
* Default Key Map Serialization and Deserialization.
*/
public class DefaultKeyMapStoreSerializer<K, S, T> extends AbstractSerialization implements
KeyMapStoreSerializer<K, S, T> {
/** Default Key Map Serialization and Deserialization. */
public class DefaultKeyMapStoreSerializer<K, S, T> extends AbstractSerialization
implements KeyMapStoreSerializer<K, S, T> {
@Override
public byte[] serializeKey(K key) {
@@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl;
import io.ray.streaming.state.serialization.KeyValueStoreSerialization;
import io.ray.streaming.state.serialization.Serializer;
/**
* KV Store Serialization and Deserialization.
*/
public class DefaultKeyValueStoreSerialization<K, V> extends AbstractSerialization implements
KeyValueStoreSerialization<K, V> {
/** KV Store Serialization and Deserialization. */
public class DefaultKeyValueStoreSerialization<K, V> extends AbstractSerialization
implements KeyValueStoreSerialization<K, V> {
@Override
public byte[] serializeKey(K key) {
@@ -21,18 +21,12 @@ package io.ray.streaming.state.store;
import java.io.IOException;
import java.util.Map;
/**
* Key Map Store interface.
*/
/** Key Map Store interface. */
public interface KeyMapStore<K, S, T> extends KeyValueStore<K, Map<S, T>> {
/**
* put sub key value into the store incrementally.
*/
/** put sub key value into the store incrementally. */
void put(K key, S subKey, T value) throws IOException;
/**
* get subValue from store.
*/
/** get subValue from store. */
T get(K key, S subKey) throws IOException;
}
@@ -21,38 +21,24 @@ package io.ray.streaming.state.store;
import java.io.IOException;
import java.io.Serializable;
/**
* Key Value Store interface.
*/
/** Key Value Store interface. */
public interface KeyValueStore<K, V> extends Serializable {
/**
* put key value into store.
*/
/** put key value into store. */
void put(K key, V value) throws IOException;
/**
* get value from store.
*/
/** get value from store. */
V get(K key) throws IOException;
/**
* remove key in the store.
*/
/** remove key in the store. */
void remove(K key) throws IOException;
/**
* flush to disk.
*/
/** flush to disk. */
void flush() throws IOException;
/**
* clear all cache.
*/
/** clear all cache. */
void clearCache();
/**
* close the store.
*/
/** close the store. */
void close() throws IOException;
}
@@ -24,9 +24,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
/**
* Memory Key Map Store.
*/
/** Memory Key Map Store. */
public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
private Map<K, Map<S, T>> memoryStore;
@@ -70,14 +68,10 @@ public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
}
@Override
public void flush() throws IOException {
}
public void flush() throws IOException {}
@Override
public void clearCache() {
}
public void clearCache() {}
@Override
public void close() throws IOException {
@@ -85,5 +79,4 @@ public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
memoryStore.clear();
}
}
}
@@ -23,9 +23,7 @@ import io.ray.streaming.state.store.KeyValueStore;
import java.io.IOException;
import java.util.Map;
/**
* Memory Key Value Store.
*/
/** Memory Key Value Store. */
public class MemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
private Map<K, V> memoryStore;
@@ -50,14 +48,10 @@ public class MemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
}
@Override
public void flush() throws IOException {
}
public void flush() throws IOException {}
@Override
public void clearCache() {
}
public void clearCache() {}
@Override
public void close() throws IOException {
@@ -33,20 +33,15 @@ import java.util.concurrent.ConcurrentHashMap;
*/
public abstract class AbstractStateStoreManager<V> implements StateStoreManager {
/**
* read-write
*/
/** read-write */
protected Map<String, StorageRecord<V>> frontStore = new ConcurrentHashMap<>();
/**
* remote-storage
*/
/** remote-storage */
protected KeyValueStore<String, Map<Long, byte[]>> kvStore;
/**
* read-only
*/
/** read-only */
protected Map<Long, Map<String, byte[]>> middleStore = new ConcurrentHashMap<>();
protected int keyGroupIndex = -1;
public AbstractStateStoreManager(KeyValueStore<String, Map<Long, byte[]>> backStore) {
@@ -31,9 +31,7 @@ import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This class define the checkpoint store strategy, which saves two-version data once.
*/
/** This class define the checkpoint store strategy, which saves two-version data once. */
public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
private static final Logger LOG = LoggerFactory.getLogger(DualStateStoreManager.class);
@@ -68,14 +66,12 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
byte[] value = entry.getValue();
/**
* 2 is specific key in kv store and indicates that new value
* should be stored with this key after overwriting old value in key 1. i.e.
* 2 is specific key in kv store and indicates that new value should be stored with this key
* after overwriting old value in key 1. i.e.
*
* -2 -1 1 2
* k1 6 5 a b
* k2 9 7 d e
* <p>-2 -1 1 2 k1 6 5 a b k2 9 7 d e
*
* k1's value for checkpoint 5 is a, and b for checkpoint 6.
* <p>k1's value for checkpoint 5 is a, and b for checkpoint 6.
*/
Map<Long, byte[]> remoteData = super.kvStore.get(key);
if (remoteData == null || remoteData.size() == 0) {
@@ -85,12 +81,12 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
} else {
long oldBatchId = Longs.fromByteArray(remoteData.get(-2L));
if (oldBatchId < checkpointId) {
//move the old data
// move the old data
remoteData.put(1L, remoteData.get(2L));
remoteData.put(-1L, remoteData.get(-2L));
}
//put the new data here
// put the new data here
remoteData.put(2L, value);
remoteData.put(-2L, Longs.toByteArray(checkpointId));
}
@@ -119,7 +115,7 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
return storageRecord.getValue();
}
//get from not commit cp info
// get from not commit cp info
List<Long> checkpointIds = new ArrayList<>(middleStore.keySet());
Collections.sort(checkpointIds);
for (int i = checkpointIds.size() - 1; i >= 0; i--) {
@@ -28,9 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* This class define the multi-version store strategy, which leverages external storage's mvcc.
*/
/** This class define the multi-version store strategy, which leverages external storage's mvcc. */
public class MVStateStoreManager<V> extends AbstractStateStoreManager<V> {
public MVStateStoreManager(KeyValueStore<String, Map<Long, byte[]>> kvStore) {
@@ -153,5 +151,4 @@ public class MVStateStoreManager<V> extends AbstractStateStoreManager<V> {
this.middleStore.remove(commitBatchId);
}
}
}
@@ -27,22 +27,20 @@ import java.util.Map;
/**
* This class support ITransactionState.
* <p>
* Based on the given StorageMode, different implementation instance of the AbstractStateStrategy
*
* <p>Based on the given StorageMode, different implementation instance of the AbstractStateStrategy
* class will be created. All method calls will be delegated to the strategy instance.
*/
public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
protected final AbstractStateStoreManager<V> stateStrategy;
private final AbstractKeyStateBackend keyStateBackend;
public StateStoreManagerProxy(
AbstractKeyStateBackend keyStateBackend,
AbstractStateDescriptor stateDescriptor) {
AbstractKeyStateBackend keyStateBackend, AbstractStateDescriptor stateDescriptor) {
this.keyStateBackend = keyStateBackend;
KeyValueStore<String, Map<Long, byte[]>> backStorage = keyStateBackend
.getBackStorage(stateDescriptor);
KeyValueStore<String, Map<Long, byte[]>> backStorage =
keyStateBackend.getBackStorage(stateDescriptor);
StateStrategy stateStrategy = keyStateBackend.getStateStrategy();
switch (stateStrategy) {
case DUAL_VERSION:
@@ -65,17 +63,13 @@ public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
this.stateStrategy.finish(checkpointId);
}
/**
* The commit can be used in another thread to reach async state commit.
*/
/** The commit can be used in another thread to reach async state commit. */
@Override
public void commit(long checkpointId) {
this.stateStrategy.commit(checkpointId);
}
/**
* The ackCommit must be called after commit in the same thread.
*/
/** The ackCommit must be called after commit in the same thread. */
@Override
public void ackCommit(long checkpointId, long timeStamp) {
this.stateStrategy.ackCommit(checkpointId);
@@ -37,8 +37,8 @@ public class KeyStateBackendTest {
public void testGetValueState() {
keyStateBackend.setCheckpointId(1L);
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("value", String.class, null);
ValueStateDescriptor<String> valueStateDescriptor =
ValueStateDescriptor.build("value", String.class, null);
valueStateDescriptor.setTableName("kepler_hlg_ut");
ValueState<String> valueState = keyStateBackend.getValueState(valueStateDescriptor);
@@ -114,13 +114,12 @@ public class KeyStateBackendTest {
Assert.assertEquals(valueState.get(), "jack");
valueState.setCurrentKey("9");
Assert.assertNull(valueState.get());
}
public void testGetListState() {
keyStateBackend.setCheckpointId(1l);
ListStateDescriptor<String> listStateDescriptor = ListStateDescriptor
.build("list", String.class);
ListStateDescriptor<String> listStateDescriptor =
ListStateDescriptor.build("list", String.class);
listStateDescriptor.setTableName("kepler_hlg_ut");
ListState<String> listState = keyStateBackend.getListState(listStateDescriptor);
@@ -207,9 +206,8 @@ public class KeyStateBackendTest {
public void testGetMapState() {
keyStateBackend.setCheckpointId(1l);
MapStateDescriptor<String, String> mapStateDescriptor = MapStateDescriptor
.build("map", String.class,
String.class);
MapStateDescriptor<String, String> mapStateDescriptor =
MapStateDescriptor.build("map", String.class, String.class);
mapStateDescriptor.setTableName("kepler_hlg_ut");
MapState<String, String> mapState = keyStateBackend.getMapState(mapStateDescriptor);
@@ -223,7 +221,7 @@ public class KeyStateBackendTest {
Assert.assertEquals(mapState.get("hello3"), null);
mapState.setCurrentKey("2");
//Assert.assertEquals(mapState.iterator(), (new HashMap()));
// Assert.assertEquals(mapState.iterator(), (new HashMap()));
mapState.setCurrentKey("2");
mapState.put("eagle", "eagle-1");
@@ -303,7 +301,6 @@ public class KeyStateBackendTest {
Assert.assertNull(mapState.get("6666"));
}
@Test
public void testMem() {
stateBackend = StateBackendBuilder.buildStateBackend(new HashMap<>());
@@ -312,5 +309,4 @@ public class KeyStateBackendTest {
testGetListState();
testGetMapState();
}
}
@@ -56,5 +56,4 @@ public class DefaultKeyMapStoreSerializationTest {
byte[] result = this.defaultKMapStoreSerDe.serializeUValue(value);
Assert.assertEquals(value, this.defaultKMapStoreSerDe.deserializeUValue(result));
}
}
@@ -58,7 +58,6 @@ public class MemoryKeyMapStoreTest {
Assert.assertNotEquals(IKeyMapStore.get("hello"), map);
Assert.assertEquals(IKeyMapStore.get("hello"), map2);
} catch (IOException e) {
e.printStackTrace();
}
@@ -20,12 +20,10 @@ package io.ray.streaming.state.keystate;
import static org.testng.Assert.assertEquals;
import org.testng.annotations.Test;
public class KeyGroupAssignmentTest {
@Test
public void testComputeKeyGroupRangeForOperatorIndex() throws Exception {
KeyGroup keyGroup = KeyGroupAssignment.getKeyGroup(4096, 1, 0);
@@ -53,5 +51,4 @@ public class KeyGroupAssignmentTest {
assertEquals(keyGroup3.getEndIndex(), 4095);
assertEquals(keyGroup3.size(), 1365);
}
}
@@ -26,8 +26,8 @@ public class ListStateDescriptorTest {
@Test
public void test() {
ListStateDescriptor<Integer> descriptor = ListStateDescriptor
.build("lsdTest", Integer.class, true);
ListStateDescriptor<Integer> descriptor =
ListStateDescriptor.build("lsdTest", Integer.class, true);
descriptor.setTableName("table");
Assert.assertTrue(descriptor.isOperatorList());
@@ -37,7 +37,8 @@ public class ListStateDescriptorTest {
Assert.assertEquals(descriptor.getPartitionNumber(), 3);
Assert.assertEquals(descriptor.getIndex(), 0);
Assert.assertEquals(descriptor.getIdentify(),
Assert.assertEquals(
descriptor.getIdentify(),
"lsdTest" + ConfigKey.DELIMITER + "3" + ConfigKey.DELIMITER + "0");
}
}
@@ -25,13 +25,11 @@ public class MapStateDescriptorTest {
@Test
public void test() {
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
.build("msdTest", String.class,
Integer.class);
MapStateDescriptor<String, Integer> descriptor =
MapStateDescriptor.build("msdTest", String.class, Integer.class);
descriptor.setTableName("table");
Assert.assertEquals(descriptor.getTableName(), "table");
Assert.assertEquals(descriptor.getName(), "msdTest");
}
}
@@ -25,8 +25,8 @@ public class ValueStateDescriptorTest {
@Test
public void test() {
ValueStateDescriptor<Integer> descriptor = ValueStateDescriptor
.build("vsdTest", Integer.class, 0);
ValueStateDescriptor<Integer> descriptor =
ValueStateDescriptor.build("vsdTest", Integer.class, 0);
Assert.assertEquals(descriptor.getDefaultValue().intValue(), 0);
}
}
@@ -36,10 +36,10 @@ public class ListStateImplTest {
@BeforeClass
public void setUp() throws Exception {
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
new MemoryStateBackend(new HashMap<>()));
ListStateDescriptor<Integer> descriptor = ListStateDescriptor
.build("ListStateImplTest", Integer.class);
keyStateBackend =
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
ListStateDescriptor<Integer> descriptor =
ListStateDescriptor.build("ListStateImplTest", Integer.class);
descriptor.setTableName("table");
listState = (ListStateImpl<Integer>) keyStateBackend.getListState(descriptor);
@@ -72,7 +72,6 @@ public class ListStateImplTest {
Assert.assertEquals(listState.get(), Arrays.asList(5, 6));
}
@Test(dependsOnMethods = {"testAddGet"})
public void testUpdate() throws Exception {
Assert.assertEquals(listState.get(), Arrays.asList(5, 6));
@@ -36,11 +36,10 @@ public class MapStateImplTest {
@BeforeClass
public void setUp() throws Exception {
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
new MemoryStateBackend(new HashMap<>()));
MapStateDescriptor<Integer, String> descriptor = MapStateDescriptor
.build("MapStateImplTest", Integer.class,
String.class);
keyStateBackend =
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
MapStateDescriptor<Integer, String> descriptor =
MapStateDescriptor.build("MapStateImplTest", Integer.class, String.class);
descriptor.setTableName("table");
mapState = (MapStateImpl<Integer, String>) keyStateBackend.getMapState(descriptor);
}
@@ -88,7 +87,6 @@ public class MapStateImplTest {
Assert.assertEquals(mapState.get().size(), 0);
}
@Test
public void testFailover() throws Exception {
keyStateBackend.setContext(1L, 1);
@@ -41,9 +41,9 @@ public class OperatorStateImplTest {
operatorStateBackend = new OperatorStateBackend(new MemoryStateBackend(config));
descriptor = ListStateDescriptor
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class,
true);
descriptor =
ListStateDescriptor.build(
"OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, true);
descriptor.setPartitionNumber(1);
descriptor.setIndex(0);
descriptor.setTableName(table_name);
@@ -34,11 +34,10 @@ public class ValueStateImplTest {
@BeforeClass
public void setUp() throws Exception {
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
new MemoryStateBackend(new HashMap<>()));
ValueStateDescriptor<String> descriptor = ValueStateDescriptor
.build("ValueStateImplTest", String.class,
"hello");
keyStateBackend =
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
ValueStateDescriptor<String> descriptor =
ValueStateDescriptor.build("ValueStateImplTest", String.class, "hello");
descriptor.setTableName("table");
valueState = (ValueStateImpl<String>) keyStateBackend.getValueState(descriptor);
@@ -31,8 +31,8 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest {
@BeforeClass
public void setUp() {
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
.build("map", String.class, Integer.class);
MapStateDescriptor<String, Integer> descriptor =
MapStateDescriptor.build("map", String.class, Integer.class);
descriptor.setTableName("tableName");
keyStateBackend.setContext(1L, "key");
facade = new MapStateStoreManagerProxy<>(keyStateBackend, descriptor);
@@ -52,5 +52,4 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest {
facade.put("key2", map);
Assert.assertEquals(facade.get("key2"), map);
}
}
@@ -25,6 +25,7 @@ import java.util.HashMap;
public class StateStoreManagerTest {
protected KeyStateBackend keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 1),
new MemoryStateBackend(new HashMap<String, String>()));
protected KeyStateBackend keyStateBackend =
new KeyStateBackend(
1, new KeyGroup(1, 1), new MemoryStateBackend(new HashMap<String, String>()));
}
@@ -30,8 +30,8 @@ public class ValueStateStoreManagerTest extends StateStoreManagerTest {
@BeforeClass
public void setUp() {
ValueStateDescriptor<Integer> descriptor = ValueStateDescriptor
.build("value", Integer.class, 0);
ValueStateDescriptor<Integer> descriptor =
ValueStateDescriptor.build("value", Integer.class, 0);
descriptor.setTableName("tableName");
keyStateBackend.setContext(1L, "key");
proxy = new ValueStateStoreManagerProxy<>(keyStateBackend, descriptor);
@@ -54,9 +54,8 @@ public class DualStateStrategyTest {
}
public void caseKV() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("VALUE-" + currentTime,
String.class, defaultValue);
ValueStateDescriptor<String> valueStateDescriptor =
ValueStateDescriptor.build("VALUE-" + currentTime, String.class, defaultValue);
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -116,7 +115,7 @@ public class DualStateStrategyTest {
this.keyStateBackend.commit(2);
this.keyStateBackend.ackCommit(2, 2);
//do rollback, all memory data is deleted.
// do rollback, all memory data is deleted.
this.keyStateBackend.rollBack(1);
this.keyStateBackend.setCheckpointId(1);
state.setCurrentKey(("1"));
@@ -159,9 +158,8 @@ public class DualStateStrategyTest {
}
public void caseKVGap() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("value2-" + currentTime,
String.class, defaultValue);
ValueStateDescriptor<String> valueStateDescriptor =
ValueStateDescriptor.build("value2-" + currentTime, String.class, defaultValue);
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -205,13 +203,11 @@ public class DualStateStrategyTest {
this.keyStateBackend.setCheckpointId(11);
this.keyStateBackend.rollBack(11);
Assert.assertEquals(state.get(), "info");
}
public void caseKList() {
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
.build("LIST-" + currentTime,
Integer.class);
ListStateDescriptor<Integer> listStateDescriptor =
ListStateDescriptor.build("LIST-" + currentTime, Integer.class);
listStateDescriptor.setTableName(table);
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
@@ -272,7 +268,7 @@ public class DualStateStrategyTest {
this.keyStateBackend.commit(2);
this.keyStateBackend.ackCommit(2, 2);
//do rollback, all memory data is deleted.
// do rollback, all memory data is deleted.
this.keyStateBackend.rollBack(1);
this.keyStateBackend.setCheckpointId(1);
state.setCurrentKey(("1"));
@@ -312,9 +308,8 @@ public class DualStateStrategyTest {
}
public void caseKMap() {
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
.build("MAP-" + currentTime,
Integer.class, Integer.class);
MapStateDescriptor<Integer, Integer> mapStateDescriptor =
MapStateDescriptor.build("MAP-" + currentTime, Integer.class, Integer.class);
mapStateDescriptor.setTableName(table);
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
@@ -378,7 +373,7 @@ public class DualStateStrategyTest {
this.keyStateBackend.commit(2);
this.keyStateBackend.ackCommit(2, 2);
//do rollback, memory data is deleted.
// do rollback, memory data is deleted.
this.keyStateBackend.rollBack(1);
this.keyStateBackend.setCheckpointId(1);
state.setCurrentKey(("1"));
@@ -428,8 +423,8 @@ public class DualStateStrategyTest {
@Test
public void testMem() {
config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name());
this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3),
StateBackendBuilder.buildStateBackend(config));
this.keyStateBackend =
new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config));
caseKV();
caseKVGap();
caseKList();
@@ -52,9 +52,8 @@ public class MVStateStrategyTest {
public void caseKV() {
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
.build("mvint-" + currentTime,
String.class, "");
ValueStateDescriptor<String> valueStateDescriptor =
ValueStateDescriptor.build("mvint-" + currentTime, String.class, "");
valueStateDescriptor.setTableName(table);
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
@@ -170,9 +169,8 @@ public class MVStateStrategyTest {
}
public void caseKList() {
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
.build("mvlist-" + currentTime,
Integer.class);
ListStateDescriptor<Integer> listStateDescriptor =
ListStateDescriptor.build("mvlist-" + currentTime, Integer.class);
listStateDescriptor.setTableName(table);
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
@@ -275,9 +273,8 @@ public class MVStateStrategyTest {
}
public void caseKMap() {
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
.build("mvmap-" + currentTime,
Integer.class, Integer.class);
MapStateDescriptor<Integer, Integer> mapStateDescriptor =
MapStateDescriptor.build("mvmap-" + currentTime, Integer.class, Integer.class);
mapStateDescriptor.setTableName(table);
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
@@ -394,15 +391,13 @@ public class MVStateStrategyTest {
Assert.assertNull(state.get(7));
}
@Test
public void testMemMV() {
config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name());
this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3),
StateBackendBuilder.buildStateBackend(config));
this.keyStateBackend =
new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config));
caseKV();
caseKList();
caseKMap();
}
}