mirror of
https://github.com/wassname/ray.git
synced 2026-07-08 08:16:42 +08:00
[Java] Format ray java code (#13056)
This commit is contained in:
+3
-9
@@ -18,18 +18,12 @@
|
||||
|
||||
package io.ray.streaming.state;
|
||||
|
||||
/**
|
||||
* Key Value State interface.
|
||||
*/
|
||||
/** Key Value State interface. */
|
||||
public interface KeyValueState<K, V> {
|
||||
|
||||
/**
|
||||
* get value from state
|
||||
*/
|
||||
/** get value from state */
|
||||
V get(K key);
|
||||
|
||||
/**
|
||||
* put key and value into state
|
||||
*/
|
||||
/** put key and value into state */
|
||||
void put(K k, V v);
|
||||
}
|
||||
|
||||
+4
-8
@@ -20,19 +20,15 @@ package io.ray.streaming.state;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* value record for partition.
|
||||
*/
|
||||
/** value record for partition. */
|
||||
public class PartitionRecord<T> implements Serializable {
|
||||
|
||||
/**
|
||||
* The partition number of the partitioned value.
|
||||
*/
|
||||
/** The partition number of the partitioned value. */
|
||||
private int partitionID;
|
||||
|
||||
private T value;
|
||||
|
||||
public PartitionRecord() {
|
||||
}
|
||||
public PartitionRecord() {}
|
||||
|
||||
public PartitionRecord(int partitionID, T value) {
|
||||
this.partitionID = partitionID;
|
||||
|
||||
+1
-3
@@ -18,9 +18,7 @@
|
||||
|
||||
package io.ray.streaming.state;
|
||||
|
||||
/**
|
||||
* RuntimeException wrapper, indicating the exceptions occurs in states.
|
||||
*/
|
||||
/** RuntimeException wrapper, indicating the exceptions occurs in states. */
|
||||
public class StateException extends RuntimeException {
|
||||
|
||||
public StateException(Throwable t) {
|
||||
|
||||
+3
-5
@@ -20,8 +20,8 @@ package io.ray.streaming.state;
|
||||
|
||||
/**
|
||||
* TransactionState interface.
|
||||
* <p>
|
||||
* Streaming State should implement transaction in case of failure, which in our case is four
|
||||
*
|
||||
* <p>Streaming State should implement transaction in case of failure, which in our case is four
|
||||
* default method, finish, commit, ackCommit, rollback.
|
||||
*/
|
||||
public interface StateStoreManager {
|
||||
@@ -44,8 +44,6 @@ public interface StateStoreManager {
|
||||
*/
|
||||
void ackCommit(long checkpointId, long timeStamp);
|
||||
|
||||
/**
|
||||
* The rollback method is used for recovering the checkpoint.
|
||||
*/
|
||||
/** The rollback method is used for recovering the checkpoint. */
|
||||
void rollBack(long checkpointId);
|
||||
}
|
||||
|
||||
+2
-5
@@ -20,16 +20,13 @@ package io.ray.streaming.state;
|
||||
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* This Class contains a record with some checkpointId.
|
||||
*/
|
||||
/** This Class contains a record with some checkpointId. */
|
||||
public class StorageRecord<T> implements Serializable {
|
||||
|
||||
private long checkpointId;
|
||||
private T value;
|
||||
|
||||
public StorageRecord() {
|
||||
}
|
||||
public StorageRecord() {}
|
||||
|
||||
public StorageRecord(long checkpointId, T value) {
|
||||
this.checkpointId = checkpointId;
|
||||
|
||||
+4
-5
@@ -36,8 +36,8 @@ import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Transaction support primitive operations like finish, commit, ackcommit and rollback.
|
||||
* <p>
|
||||
* State value modification is not thread safe! By default, every processing thread has its own
|
||||
*
|
||||
* <p>State value modification is not thread safe! By default, every processing thread has its own
|
||||
* space to handle state.
|
||||
*/
|
||||
public abstract class AbstractKeyStateBackend implements StateStoreManager {
|
||||
@@ -52,10 +52,9 @@ public abstract class AbstractKeyStateBackend implements StateStoreManager {
|
||||
protected Map<String, MapStateStoreManagerProxy> mapManagerProxyHashMap = new HashMap<>();
|
||||
protected Set<String> descNamespace;
|
||||
|
||||
/**
|
||||
* tablename, KeyValueStore key, checkpointId, content
|
||||
*/
|
||||
/** tablename, KeyValueStore key, checkpointId, content */
|
||||
protected Map<String, KeyValueStore<String, Map<Long, byte[]>>> backStorageCache;
|
||||
|
||||
private AbstractStateBackend backend;
|
||||
|
||||
public AbstractKeyStateBackend(AbstractStateBackend backend) {
|
||||
|
||||
+1
-4
@@ -20,7 +20,6 @@ package io.ray.streaming.state.backend;
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import io.ray.streaming.state.config.ConfigKey;
|
||||
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
|
||||
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
|
||||
@@ -29,9 +28,7 @@ import io.ray.streaming.state.store.KeyValueStore;
|
||||
import java.io.Serializable;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* This class is the abstract class for different kinds of StateBackend.
|
||||
*/
|
||||
/** This class is the abstract class for different kinds of StateBackend. */
|
||||
public abstract class AbstractStateBackend implements Serializable {
|
||||
|
||||
protected final Map<String, String> config;
|
||||
|
||||
+3
-9
@@ -18,18 +18,12 @@
|
||||
|
||||
package io.ray.streaming.state.backend;
|
||||
|
||||
/**
|
||||
* Backend Types.
|
||||
*/
|
||||
/** Backend Types. */
|
||||
public enum BackendType {
|
||||
/**
|
||||
* Saving the state values in memory
|
||||
*/
|
||||
/** Saving the state values in memory */
|
||||
MEMORY;
|
||||
|
||||
/**
|
||||
* get the enum from input string value, ignoring the case
|
||||
*/
|
||||
/** get the enum from input string value, ignoring the case */
|
||||
public static BackendType getEnum(String value) {
|
||||
for (BackendType v : values()) {
|
||||
if (v.name().equalsIgnoreCase(value)) {
|
||||
|
||||
+6
-13
@@ -40,16 +40,13 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
|
||||
protected final KeyGroup keyGroup;
|
||||
|
||||
public KeyStateBackend(
|
||||
int numberOfKeyGroups, KeyGroup keyGroup,
|
||||
AbstractStateBackend abstractStateBackend) {
|
||||
int numberOfKeyGroups, KeyGroup keyGroup, AbstractStateBackend abstractStateBackend) {
|
||||
super(abstractStateBackend);
|
||||
this.numberOfKeyGroups = numberOfKeyGroups;
|
||||
this.keyGroup = keyGroup;
|
||||
}
|
||||
|
||||
/**
|
||||
* get new value state proxy
|
||||
*/
|
||||
/** get new value state proxy */
|
||||
protected <T> ValueStateStoreManagerProxy<T> newValueStateProxy(
|
||||
ValueStateDescriptor<T> stateDescriptor) {
|
||||
return new ValueStateStoreManagerProxy<>(this, stateDescriptor);
|
||||
@@ -66,9 +63,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get new list state proxy
|
||||
*/
|
||||
/** get new list state proxy */
|
||||
protected <T> ListStateStoreManagerProxy<T> newListStateProxy(
|
||||
ListStateDescriptor<T> stateDescriptor) {
|
||||
return new ListStateStoreManagerProxy<>(this, stateDescriptor);
|
||||
@@ -86,9 +81,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get map state proxy
|
||||
*/
|
||||
/** get map state proxy */
|
||||
protected <S, T> MapStateStoreManagerProxy<S, T> newMapStateProxy(
|
||||
MapStateDescriptor<S, T> stateDescriptor) {
|
||||
return new MapStateStoreManagerProxy<>(this, stateDescriptor);
|
||||
@@ -108,8 +101,8 @@ public class KeyStateBackend extends AbstractKeyStateBackend {
|
||||
|
||||
@Override
|
||||
public void setCurrentKey(Object currentKey) {
|
||||
super.keyGroupIndex = KeyGroupAssignment
|
||||
.assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups);
|
||||
super.keyGroupIndex =
|
||||
KeyGroupAssignment.assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups);
|
||||
super.currentKey = currentKey;
|
||||
}
|
||||
|
||||
|
||||
+3
-9
@@ -23,9 +23,7 @@ import io.ray.streaming.state.keystate.state.ListState;
|
||||
import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl;
|
||||
import io.ray.streaming.state.keystate.state.proxy.ListStateStoreManagerProxy;
|
||||
|
||||
/**
|
||||
* OperatorState manager for getting split or union list state.
|
||||
*/
|
||||
/** OperatorState manager for getting split or union list state. */
|
||||
public class OperatorStateBackend extends AbstractKeyStateBackend {
|
||||
|
||||
public OperatorStateBackend(AbstractStateBackend backend) {
|
||||
@@ -42,9 +40,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend {
|
||||
return new ListStateStoreManagerProxy<>(this, stateDescriptor);
|
||||
}
|
||||
|
||||
/**
|
||||
* get spitted List for different operator instance.
|
||||
*/
|
||||
/** get spitted List for different operator instance. */
|
||||
public <T> ListState<T> getSplitListState(ListStateDescriptor<T> stateDescriptor) {
|
||||
String desc = stateDescriptor.getIdentify();
|
||||
if (listManagerProxyHashMap.containsKey(desc)) {
|
||||
@@ -58,9 +54,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get a union List for different operator instance.
|
||||
*/
|
||||
/** get a union List for different operator instance. */
|
||||
public <T> ListState<T> getUnionListState(ListStateDescriptor<T> stateDescriptor) {
|
||||
String desc = stateDescriptor.getIdentify();
|
||||
if (listManagerProxyHashMap.containsKey(desc)) {
|
||||
|
||||
+2
-5
@@ -22,14 +22,11 @@ import io.ray.streaming.state.backend.impl.MemoryStateBackend;
|
||||
import io.ray.streaming.state.config.ConfigKey;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* State Backend Builder.
|
||||
*/
|
||||
/** State Backend Builder. */
|
||||
public class StateBackendBuilder {
|
||||
|
||||
private static AbstractStateBackend getStateBackend(
|
||||
Map<String, String> config,
|
||||
BackendType type) {
|
||||
Map<String, String> config, BackendType type) {
|
||||
switch (type) {
|
||||
case MEMORY:
|
||||
return new MemoryStateBackend(config);
|
||||
|
||||
+3
-10
@@ -18,21 +18,14 @@
|
||||
|
||||
package io.ray.streaming.state.backend;
|
||||
|
||||
/**
|
||||
* This class describe State Saving Model.
|
||||
*/
|
||||
/** This class describe State Saving Model. */
|
||||
public enum StateStrategy {
|
||||
/**
|
||||
* save two version together in case of rollback.
|
||||
*/
|
||||
/** save two version together in case of rollback. */
|
||||
DUAL_VERSION,
|
||||
|
||||
/**
|
||||
* for storage supporting mvcc, we save only current version.
|
||||
*/
|
||||
/** for storage supporting mvcc, we save only current version. */
|
||||
SINGLE_VERSION;
|
||||
|
||||
|
||||
public static StateStrategy getEnum(String value) {
|
||||
for (StateStrategy v : values()) {
|
||||
if (v.name().equalsIgnoreCase(value)) {
|
||||
|
||||
+1
-4
@@ -26,9 +26,7 @@ import io.ray.streaming.state.store.impl.MemoryKeyMapStore;
|
||||
import io.ray.streaming.state.store.impl.MemoryKeyValueStore;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* MemoryStateBackend. Supporting memory store.
|
||||
*/
|
||||
/** MemoryStateBackend. Supporting memory store. */
|
||||
public class MemoryStateBackend extends AbstractStateBackend {
|
||||
|
||||
public MemoryStateBackend(Map<String, String> config) {
|
||||
@@ -50,5 +48,4 @@ public class MemoryStateBackend extends AbstractStateBackend {
|
||||
String tableName, KeyMapStoreSerializer keyMapStoreSerializer) {
|
||||
return new MemoryKeyMapStore<>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-4
@@ -20,9 +20,7 @@ package io.ray.streaming.state.config;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Config Helper figure out the config info. Todo replace this to config module.
|
||||
*/
|
||||
/** Config Helper figure out the config info. Todo replace this to config module. */
|
||||
public class ConfigHelper {
|
||||
|
||||
public static int getIntegerOrDefault(Map config, String configKey, int defaultValue) {
|
||||
@@ -40,5 +38,4 @@ public class ConfigHelper {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-6
@@ -20,15 +20,12 @@ package io.ray.streaming.state.config;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* state config keys. Todo replace this to config module.
|
||||
*/
|
||||
/** state config keys. Todo replace this to config module. */
|
||||
public final class ConfigKey {
|
||||
|
||||
/**
|
||||
* backend
|
||||
*/
|
||||
/** backend */
|
||||
public static final String STATE_BACKEND_TYPE = "state.backend.type";
|
||||
|
||||
public static final String STATE_TABLE_NAME = "state.table.name";
|
||||
public static final String STATE_STRATEGY_MODE = "state.strategy.mode";
|
||||
public static final String NUMBER_PER_CHECKPOINT = "number.per.checkpoint";
|
||||
|
||||
+1
-3
@@ -43,9 +43,7 @@ public class KeyGroup implements Serializable {
|
||||
Preconditions.checkArgument(size() >= 0, "overflow detected.");
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The number of key-group in the range
|
||||
*/
|
||||
/** Returns The number of key-group in the range */
|
||||
public int size() {
|
||||
return 1 + endIndex - startIndex;
|
||||
}
|
||||
|
||||
+10
-11
@@ -24,9 +24,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* This class defines key-group assignment algorithm。
|
||||
*/
|
||||
/** This class defines key-group assignment algorithm。 */
|
||||
public final class KeyGroupAssignment {
|
||||
|
||||
/**
|
||||
@@ -37,8 +35,10 @@ public final class KeyGroupAssignment {
|
||||
* @param index index of the operator instance.
|
||||
*/
|
||||
public static KeyGroup getKeyGroup(int maxParallelism, int parallelism, int index) {
|
||||
Preconditions.checkArgument(maxParallelism >= parallelism,
|
||||
"Maximum parallelism (%s) must not be smaller than parallelism(%s)", maxParallelism,
|
||||
Preconditions.checkArgument(
|
||||
maxParallelism >= parallelism,
|
||||
"Maximum parallelism (%s) must not be smaller than parallelism(%s)",
|
||||
maxParallelism,
|
||||
parallelism);
|
||||
|
||||
int start = index == 0 ? 0 : ((index * maxParallelism - 1) / parallelism) + 1;
|
||||
@@ -50,25 +50,24 @@ public final class KeyGroupAssignment {
|
||||
* Assigning the key to a key-group index.
|
||||
*
|
||||
* @param key the key to assign.
|
||||
* @param maxParallelism the maximum parallelism.
|
||||
* @return the key-group index to which the given key is assigned.
|
||||
* @param maxParallelism the maximum parallelism. Returns the key-group index to which the given
|
||||
* key is assigned.
|
||||
*/
|
||||
public static int assignKeyGroupIndexForKey(Object key, int maxParallelism) {
|
||||
return Math.abs(key.hashCode() % maxParallelism);
|
||||
}
|
||||
|
||||
public static Map<Integer, List<Integer>> computeKeyGroupToTask(
|
||||
int maxParallelism,
|
||||
List<Integer> targetTasks) {
|
||||
int maxParallelism, List<Integer> targetTasks) {
|
||||
Map<Integer, List<Integer>> keyGroupToTask = new ConcurrentHashMap<>();
|
||||
for (int index = 0; index < targetTasks.size(); index++) {
|
||||
KeyGroup taskKeyGroup = getKeyGroup(maxParallelism, targetTasks.size(), index);
|
||||
for (int groupId = taskKeyGroup.getStartIndex(); groupId <= taskKeyGroup.getEndIndex();
|
||||
for (int groupId = taskKeyGroup.getStartIndex();
|
||||
groupId <= taskKeyGroup.getEndIndex();
|
||||
groupId++) {
|
||||
keyGroupToTask.put(groupId, ImmutableList.of(targetTasks.get(index)));
|
||||
}
|
||||
}
|
||||
return keyGroupToTask;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+14
-14
@@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc;
|
||||
import com.google.common.base.Preconditions;
|
||||
import io.ray.streaming.state.keystate.state.State;
|
||||
|
||||
/**
|
||||
* This class defines basic data structures of StateDescriptor.
|
||||
*/
|
||||
/** This class defines basic data structures of StateDescriptor. */
|
||||
public abstract class AbstractStateDescriptor<S extends State, T> {
|
||||
|
||||
private final String name;
|
||||
@@ -65,24 +63,26 @@ public abstract class AbstractStateDescriptor<S extends State, T> {
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "AbstractStateDescriptor{" + "tableName='" + tableName + '\'' + ", name='" + name + '\''
|
||||
+ ", type=" + type + '}';
|
||||
return "AbstractStateDescriptor{"
|
||||
+ "tableName='"
|
||||
+ tableName
|
||||
+ '\''
|
||||
+ ", name='"
|
||||
+ name
|
||||
+ '\''
|
||||
+ ", type="
|
||||
+ type
|
||||
+ '}';
|
||||
}
|
||||
|
||||
public enum StateType {
|
||||
/**
|
||||
* value state
|
||||
*/
|
||||
/** value state */
|
||||
VALUE,
|
||||
|
||||
/**
|
||||
* list state
|
||||
*/
|
||||
/** list state */
|
||||
LIST,
|
||||
|
||||
/**
|
||||
* map state
|
||||
*/
|
||||
/** map state */
|
||||
MAP
|
||||
}
|
||||
}
|
||||
|
||||
+4
-10
@@ -18,15 +18,11 @@
|
||||
|
||||
package io.ray.streaming.state.keystate.desc;
|
||||
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import io.ray.streaming.state.keystate.state.ListState;
|
||||
|
||||
/**
|
||||
* ListStateDescriptor.
|
||||
*/
|
||||
/** ListStateDescriptor. */
|
||||
public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>, T> {
|
||||
|
||||
private final boolean isOperatorList;
|
||||
@@ -43,8 +39,7 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
|
||||
}
|
||||
|
||||
public static <T> ListStateDescriptor<T> build(
|
||||
String name, Class<T> type,
|
||||
boolean isOperatorList) {
|
||||
String name, Class<T> type, boolean isOperatorList) {
|
||||
return new ListStateDescriptor<>(name, type, isOperatorList);
|
||||
}
|
||||
|
||||
@@ -76,9 +71,8 @@ public class ListStateDescriptor<T> extends AbstractStateDescriptor<ListState<T>
|
||||
@Override
|
||||
public String getIdentify() {
|
||||
if (isOperatorList) {
|
||||
return String
|
||||
.format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER,
|
||||
index);
|
||||
return String.format(
|
||||
"%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, index);
|
||||
} else {
|
||||
return super.getIdentify();
|
||||
}
|
||||
|
||||
+2
-5
@@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc;
|
||||
import io.ray.streaming.state.keystate.state.MapState;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* MapStateDescriptor.
|
||||
*/
|
||||
/** MapStateDescriptor. */
|
||||
public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K, V>, Map<K, V>> {
|
||||
|
||||
public MapStateDescriptor(String name, Class<K> keyType, Class<V> valueType) {
|
||||
@@ -32,8 +30,7 @@ public class MapStateDescriptor<K, V> extends AbstractStateDescriptor<MapState<K
|
||||
}
|
||||
|
||||
public static <K, V> MapStateDescriptor<K, V> build(
|
||||
String name, Class<K> keyType,
|
||||
Class<V> valueType) {
|
||||
String name, Class<K> keyType, Class<V> valueType) {
|
||||
return new MapStateDescriptor<>(name, keyType, valueType);
|
||||
}
|
||||
|
||||
|
||||
+1
-3
@@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.desc;
|
||||
|
||||
import io.ray.streaming.state.keystate.state.ValueState;
|
||||
|
||||
/**
|
||||
* ValueStateDescriptor.
|
||||
*/
|
||||
/** ValueStateDescriptor. */
|
||||
public class ValueStateDescriptor<T> extends AbstractStateDescriptor<ValueState<T>, T> {
|
||||
|
||||
private final T defaultValue;
|
||||
|
||||
+1
-3
@@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.state;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* ListState interface.
|
||||
*/
|
||||
/** ListState interface. */
|
||||
public interface ListState<T> extends UnaryState<List<T>> {
|
||||
|
||||
/**
|
||||
|
||||
+8
-11
@@ -22,16 +22,13 @@ import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
/**
|
||||
* MapState interface.
|
||||
*/
|
||||
/** MapState interface. */
|
||||
public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
|
||||
/**
|
||||
* Returns the current value associated with the given key.
|
||||
*
|
||||
* @param key The key of the mapping
|
||||
* @return The value of the mapping with the given key
|
||||
* @param key The key of the mapping Returns The value of the mapping with the given key
|
||||
*/
|
||||
V get(K key);
|
||||
|
||||
@@ -67,8 +64,8 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
/**
|
||||
* Returns whether there exists the given mapping.
|
||||
*
|
||||
* @param key The key of the mapping
|
||||
* @return True if there exists a mapping whose key equals to the given key
|
||||
* @param key The key of the mapping Returns True if there exists a mapping whose key equals to
|
||||
* the given key
|
||||
*/
|
||||
default boolean contains(K key) {
|
||||
return get().containsKey(key);
|
||||
@@ -77,7 +74,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
/**
|
||||
* Returns all the mappings in the state
|
||||
*
|
||||
* @return An iterable view of all the key-value pairs in the state.
|
||||
* <p>Returns An iterable view of all the key-value pairs in the state.
|
||||
*/
|
||||
default Iterable<Entry<K, V>> entries() {
|
||||
return get().entrySet();
|
||||
@@ -86,7 +83,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
/**
|
||||
* Returns all the keys in the state
|
||||
*
|
||||
* @return An iterable view of all the keys in the state.
|
||||
* <p>Returns An iterable view of all the keys in the state.
|
||||
*/
|
||||
default Iterable<K> keys() {
|
||||
return get().keySet();
|
||||
@@ -95,7 +92,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
/**
|
||||
* Returns all the values in the state.
|
||||
*
|
||||
* @return An iterable view of all the values in the state.
|
||||
* <p>Returns An iterable view of all the values in the state.
|
||||
*/
|
||||
default Iterable<V> values() {
|
||||
return get().values();
|
||||
@@ -104,7 +101,7 @@ public interface MapState<K, V> extends UnaryState<Map<K, V>> {
|
||||
/**
|
||||
* Iterates over all the mappings in the state.
|
||||
*
|
||||
* @return An iterator over all the mappings in the state
|
||||
* <p>Returns An iterator over all the mappings in the state
|
||||
*/
|
||||
default Iterator<Entry<K, V>> iterator() {
|
||||
return get().entrySet().iterator();
|
||||
|
||||
+2
-6
@@ -18,13 +18,9 @@
|
||||
|
||||
package io.ray.streaming.state.keystate.state;
|
||||
|
||||
/**
|
||||
* State interface.
|
||||
*/
|
||||
/** State interface. */
|
||||
public interface State {
|
||||
|
||||
/**
|
||||
* set current key of the state
|
||||
*/
|
||||
/** set current key of the state */
|
||||
void setCurrentKey(Object currentKey);
|
||||
}
|
||||
|
||||
+3
-5
@@ -18,15 +18,13 @@
|
||||
|
||||
package io.ray.streaming.state.keystate.state;
|
||||
|
||||
/**
|
||||
* one value per state.
|
||||
*/
|
||||
/** one value per state. */
|
||||
public interface UnaryState<O> extends State {
|
||||
|
||||
/**
|
||||
* get the value in state
|
||||
*
|
||||
* @return the value in state
|
||||
* <p>Returns the value in state
|
||||
*/
|
||||
O get();
|
||||
}
|
||||
}
|
||||
|
||||
+1
-3
@@ -18,9 +18,7 @@
|
||||
|
||||
package io.ray.streaming.state.keystate.state;
|
||||
|
||||
/**
|
||||
* ValueState interface.
|
||||
*/
|
||||
/** ValueState interface. */
|
||||
public interface ValueState<T> extends UnaryState<T> {
|
||||
|
||||
/**
|
||||
|
||||
+1
-3
@@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.ListState;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* ListState implementation.
|
||||
*/
|
||||
/** ListState implementation. */
|
||||
public class ListStateImpl<V> implements ListState<V> {
|
||||
|
||||
private final StateHelper<List<V>> helper;
|
||||
|
||||
+2
-6
@@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.MapState;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* MapState implementation.
|
||||
*/
|
||||
/** MapState implementation. */
|
||||
public class MapStateImpl<K, V> implements MapState<K, V> {
|
||||
|
||||
private final StateHelper<Map<K, V>> helper;
|
||||
@@ -82,9 +80,7 @@ public class MapStateImpl<K, V> implements MapState<K, V> {
|
||||
helper.put(map);
|
||||
}
|
||||
|
||||
/**
|
||||
* set current key of the state
|
||||
*/
|
||||
/** set current key of the state */
|
||||
@Override
|
||||
public void setCurrentKey(Object currentKey) {
|
||||
helper.setCurrentKey(currentKey);
|
||||
|
||||
+2
-3
@@ -20,7 +20,6 @@ package io.ray.streaming.state.keystate.state.impl;
|
||||
|
||||
import static io.ray.streaming.state.config.ConfigKey.DELIMITER;
|
||||
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import io.ray.streaming.state.PartitionRecord;
|
||||
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
|
||||
@@ -73,8 +72,8 @@ public class OperatorStateImpl<V> implements ListState<V> {
|
||||
int partitionNum = -1;
|
||||
int index = 0;
|
||||
while (true) {
|
||||
List<PartitionRecord<V>> list = helper.getBackend()
|
||||
.get(descriptor, getKey(descriptor.getIdentify(), index));
|
||||
List<PartitionRecord<V>> list =
|
||||
helper.getBackend().get(descriptor, getKey(descriptor.getIdentify(), index));
|
||||
if (list != null && !list.isEmpty()) {
|
||||
partitionNum = list.get(0).getPartitionID();
|
||||
allList.addAll(list);
|
||||
|
||||
+1
-3
@@ -22,9 +22,7 @@ import com.google.common.base.Preconditions;
|
||||
import io.ray.streaming.state.backend.AbstractKeyStateBackend;
|
||||
import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor;
|
||||
|
||||
/**
|
||||
* State Helper Class.
|
||||
*/
|
||||
/** State Helper Class. */
|
||||
public class StateHelper<T> {
|
||||
|
||||
private final AbstractKeyStateBackend backend;
|
||||
|
||||
+2
-6
@@ -22,9 +22,7 @@ import io.ray.streaming.state.backend.KeyStateBackend;
|
||||
import io.ray.streaming.state.keystate.desc.ValueStateDescriptor;
|
||||
import io.ray.streaming.state.keystate.state.ValueState;
|
||||
|
||||
/**
|
||||
* ValueState implementation.
|
||||
*/
|
||||
/** ValueState implementation. */
|
||||
public class ValueStateImpl<T> implements ValueState<T> {
|
||||
|
||||
private final StateHelper<T> helper;
|
||||
@@ -48,9 +46,7 @@ public class ValueStateImpl<T> implements ValueState<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* set current key of the state
|
||||
*/
|
||||
/** set current key of the state */
|
||||
@Override
|
||||
public void setCurrentKey(Object currentKey) {
|
||||
helper.setCurrentKey(currentKey);
|
||||
|
||||
+4
-7
@@ -27,17 +27,14 @@ import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl;
|
||||
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* This class defines ListState Wrapper, connecting state and backend.
|
||||
*/
|
||||
public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T>> implements
|
||||
KeyValueState<String, List<T>> {
|
||||
/** This class defines ListState Wrapper, connecting state and backend. */
|
||||
public class ListStateStoreManagerProxy<T> extends StateStoreManagerProxy<List<T>>
|
||||
implements KeyValueState<String, List<T>> {
|
||||
|
||||
private final ListState<T> listState;
|
||||
|
||||
public ListStateStoreManagerProxy(
|
||||
AbstractKeyStateBackend keyStateBackend,
|
||||
ListStateDescriptor<T> stateDescriptor) {
|
||||
AbstractKeyStateBackend keyStateBackend, ListStateDescriptor<T> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
if (stateDescriptor.isOperatorList()) {
|
||||
this.listState = new OperatorStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
|
||||
+4
-7
@@ -26,17 +26,14 @@ import io.ray.streaming.state.keystate.state.impl.MapStateImpl;
|
||||
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* This class defines MapState Wrapper, connecting state and backend.
|
||||
*/
|
||||
public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<K, V>> implements
|
||||
KeyValueState<String, Map<K, V>> {
|
||||
/** This class defines MapState Wrapper, connecting state and backend. */
|
||||
public class MapStateStoreManagerProxy<K, V> extends StateStoreManagerProxy<Map<K, V>>
|
||||
implements KeyValueState<String, Map<K, V>> {
|
||||
|
||||
private final MapStateImpl<K, V> mapState;
|
||||
|
||||
public MapStateStoreManagerProxy(
|
||||
KeyStateBackend keyStateBackend,
|
||||
MapStateDescriptor<K, V> stateDescriptor) {
|
||||
KeyStateBackend keyStateBackend, MapStateDescriptor<K, V> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
this.mapState = new MapStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
}
|
||||
|
||||
+4
-7
@@ -25,17 +25,14 @@ import io.ray.streaming.state.keystate.state.ValueState;
|
||||
import io.ray.streaming.state.keystate.state.impl.ValueStateImpl;
|
||||
import io.ray.streaming.state.strategy.StateStoreManagerProxy;
|
||||
|
||||
/**
|
||||
* This class defines ValueState Wrapper, connecting state and backend.
|
||||
*/
|
||||
public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T> implements
|
||||
KeyValueState<String, T> {
|
||||
/** This class defines ValueState Wrapper, connecting state and backend. */
|
||||
public class ValueStateStoreManagerProxy<T> extends StateStoreManagerProxy<T>
|
||||
implements KeyValueState<String, T> {
|
||||
|
||||
private final ValueStateImpl<T> valueState;
|
||||
|
||||
public ValueStateStoreManagerProxy(
|
||||
KeyStateBackend keyStateBackend,
|
||||
ValueStateDescriptor<T> stateDescriptor) {
|
||||
KeyStateBackend keyStateBackend, ValueStateDescriptor<T> stateDescriptor) {
|
||||
super(keyStateBackend, stateDescriptor);
|
||||
this.valueState = new ValueStateImpl<>(stateDescriptor, keyStateBackend);
|
||||
}
|
||||
|
||||
+1
-4
@@ -18,9 +18,7 @@
|
||||
|
||||
package io.ray.streaming.state.serialization;
|
||||
|
||||
/**
|
||||
* Key Map Serialization and Deserialization.
|
||||
*/
|
||||
/** Key Map Serialization and Deserialization. */
|
||||
public interface KeyMapStoreSerializer<K, S, T> {
|
||||
|
||||
byte[] serializeKey(K key);
|
||||
@@ -32,5 +30,4 @@ public interface KeyMapStoreSerializer<K, S, T> {
|
||||
byte[] serializeUValue(T uv);
|
||||
|
||||
T deserializeUValue(byte[] uvArray);
|
||||
|
||||
}
|
||||
|
||||
+1
-3
@@ -18,9 +18,7 @@
|
||||
|
||||
package io.ray.streaming.state.serialization;
|
||||
|
||||
/**
|
||||
* Key Value Serialization and Deserialization.
|
||||
*/
|
||||
/** Key Value Serialization and Deserialization. */
|
||||
public interface KeyValueStoreSerialization<K, V> {
|
||||
|
||||
byte[] serializeKey(K key);
|
||||
|
||||
+1
-3
@@ -20,9 +20,7 @@ package io.ray.streaming.state.serialization;
|
||||
|
||||
import org.nustaq.serialization.FSTConfiguration;
|
||||
|
||||
/**
|
||||
* fst wrapper.
|
||||
*/
|
||||
/** fst wrapper. */
|
||||
public class Serializer {
|
||||
|
||||
private static final ThreadLocal<FSTConfiguration> conf =
|
||||
|
||||
+1
-3
@@ -24,9 +24,7 @@ import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* AbstractSerialization. Generate row key.
|
||||
*/
|
||||
/** AbstractSerialization. Generate row key. */
|
||||
public abstract class AbstractSerialization {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractSerialization.class);
|
||||
|
||||
+3
-5
@@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl;
|
||||
import io.ray.streaming.state.serialization.KeyMapStoreSerializer;
|
||||
import io.ray.streaming.state.serialization.Serializer;
|
||||
|
||||
/**
|
||||
* Default Key Map Serialization and Deserialization.
|
||||
*/
|
||||
public class DefaultKeyMapStoreSerializer<K, S, T> extends AbstractSerialization implements
|
||||
KeyMapStoreSerializer<K, S, T> {
|
||||
/** Default Key Map Serialization and Deserialization. */
|
||||
public class DefaultKeyMapStoreSerializer<K, S, T> extends AbstractSerialization
|
||||
implements KeyMapStoreSerializer<K, S, T> {
|
||||
|
||||
@Override
|
||||
public byte[] serializeKey(K key) {
|
||||
|
||||
+3
-5
@@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl;
|
||||
import io.ray.streaming.state.serialization.KeyValueStoreSerialization;
|
||||
import io.ray.streaming.state.serialization.Serializer;
|
||||
|
||||
/**
|
||||
* KV Store Serialization and Deserialization.
|
||||
*/
|
||||
public class DefaultKeyValueStoreSerialization<K, V> extends AbstractSerialization implements
|
||||
KeyValueStoreSerialization<K, V> {
|
||||
/** KV Store Serialization and Deserialization. */
|
||||
public class DefaultKeyValueStoreSerialization<K, V> extends AbstractSerialization
|
||||
implements KeyValueStoreSerialization<K, V> {
|
||||
|
||||
@Override
|
||||
public byte[] serializeKey(K key) {
|
||||
|
||||
+3
-9
@@ -21,18 +21,12 @@ package io.ray.streaming.state.store;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Key Map Store interface.
|
||||
*/
|
||||
/** Key Map Store interface. */
|
||||
public interface KeyMapStore<K, S, T> extends KeyValueStore<K, Map<S, T>> {
|
||||
|
||||
/**
|
||||
* put sub key value into the store incrementally.
|
||||
*/
|
||||
/** put sub key value into the store incrementally. */
|
||||
void put(K key, S subKey, T value) throws IOException;
|
||||
|
||||
/**
|
||||
* get subValue from store.
|
||||
*/
|
||||
/** get subValue from store. */
|
||||
T get(K key, S subKey) throws IOException;
|
||||
}
|
||||
|
||||
+7
-21
@@ -21,38 +21,24 @@ package io.ray.streaming.state.store;
|
||||
import java.io.IOException;
|
||||
import java.io.Serializable;
|
||||
|
||||
/**
|
||||
* Key Value Store interface.
|
||||
*/
|
||||
/** Key Value Store interface. */
|
||||
public interface KeyValueStore<K, V> extends Serializable {
|
||||
|
||||
/**
|
||||
* put key value into store.
|
||||
*/
|
||||
/** put key value into store. */
|
||||
void put(K key, V value) throws IOException;
|
||||
|
||||
/**
|
||||
* get value from store.
|
||||
*/
|
||||
/** get value from store. */
|
||||
V get(K key) throws IOException;
|
||||
|
||||
/**
|
||||
* remove key in the store.
|
||||
*/
|
||||
/** remove key in the store. */
|
||||
void remove(K key) throws IOException;
|
||||
|
||||
/**
|
||||
* flush to disk.
|
||||
*/
|
||||
/** flush to disk. */
|
||||
void flush() throws IOException;
|
||||
|
||||
/**
|
||||
* clear all cache.
|
||||
*/
|
||||
/** clear all cache. */
|
||||
void clearCache();
|
||||
|
||||
/**
|
||||
* close the store.
|
||||
*/
|
||||
/** close the store. */
|
||||
void close() throws IOException;
|
||||
}
|
||||
|
||||
+3
-10
@@ -24,9 +24,7 @@ import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Memory Key Map Store.
|
||||
*/
|
||||
/** Memory Key Map Store. */
|
||||
public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
|
||||
|
||||
private Map<K, Map<S, T>> memoryStore;
|
||||
@@ -70,14 +68,10 @@ public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
|
||||
}
|
||||
public void flush() throws IOException {}
|
||||
|
||||
@Override
|
||||
public void clearCache() {
|
||||
|
||||
}
|
||||
public void clearCache() {}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
@@ -85,5 +79,4 @@ public class MemoryKeyMapStore<K, S, T> implements KeyMapStore<K, S, T> {
|
||||
memoryStore.clear();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-9
@@ -23,9 +23,7 @@ import io.ray.streaming.state.store.KeyValueStore;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Memory Key Value Store.
|
||||
*/
|
||||
/** Memory Key Value Store. */
|
||||
public class MemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
|
||||
|
||||
private Map<K, V> memoryStore;
|
||||
@@ -50,14 +48,10 @@ public class MemoryKeyValueStore<K, V> implements KeyValueStore<K, V> {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush() throws IOException {
|
||||
|
||||
}
|
||||
public void flush() throws IOException {}
|
||||
|
||||
@Override
|
||||
public void clearCache() {
|
||||
|
||||
}
|
||||
public void clearCache() {}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
|
||||
+4
-9
@@ -33,20 +33,15 @@ import java.util.concurrent.ConcurrentHashMap;
|
||||
*/
|
||||
public abstract class AbstractStateStoreManager<V> implements StateStoreManager {
|
||||
|
||||
/**
|
||||
* read-write
|
||||
*/
|
||||
/** read-write */
|
||||
protected Map<String, StorageRecord<V>> frontStore = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* remote-storage
|
||||
*/
|
||||
/** remote-storage */
|
||||
protected KeyValueStore<String, Map<Long, byte[]>> kvStore;
|
||||
|
||||
/**
|
||||
* read-only
|
||||
*/
|
||||
/** read-only */
|
||||
protected Map<Long, Map<String, byte[]>> middleStore = new ConcurrentHashMap<>();
|
||||
|
||||
protected int keyGroupIndex = -1;
|
||||
|
||||
public AbstractStateStoreManager(KeyValueStore<String, Map<Long, byte[]>> backStore) {
|
||||
|
||||
+8
-12
@@ -31,9 +31,7 @@ import java.util.Map.Entry;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* This class define the checkpoint store strategy, which saves two-version data once.
|
||||
*/
|
||||
/** This class define the checkpoint store strategy, which saves two-version data once. */
|
||||
public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(DualStateStoreManager.class);
|
||||
@@ -68,14 +66,12 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
byte[] value = entry.getValue();
|
||||
|
||||
/**
|
||||
* 2 is specific key in kv store and indicates that new value
|
||||
* should be stored with this key after overwriting old value in key 1. i.e.
|
||||
* 2 is specific key in kv store and indicates that new value should be stored with this key
|
||||
* after overwriting old value in key 1. i.e.
|
||||
*
|
||||
* -2 -1 1 2
|
||||
* k1 6 5 a b
|
||||
* k2 9 7 d e
|
||||
* <p>-2 -1 1 2 k1 6 5 a b k2 9 7 d e
|
||||
*
|
||||
* k1's value for checkpoint 5 is a, and b for checkpoint 6.
|
||||
* <p>k1's value for checkpoint 5 is a, and b for checkpoint 6.
|
||||
*/
|
||||
Map<Long, byte[]> remoteData = super.kvStore.get(key);
|
||||
if (remoteData == null || remoteData.size() == 0) {
|
||||
@@ -85,12 +81,12 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
} else {
|
||||
long oldBatchId = Longs.fromByteArray(remoteData.get(-2L));
|
||||
if (oldBatchId < checkpointId) {
|
||||
//move the old data
|
||||
// move the old data
|
||||
remoteData.put(1L, remoteData.get(2L));
|
||||
remoteData.put(-1L, remoteData.get(-2L));
|
||||
}
|
||||
|
||||
//put the new data here
|
||||
// put the new data here
|
||||
remoteData.put(2L, value);
|
||||
remoteData.put(-2L, Longs.toByteArray(checkpointId));
|
||||
}
|
||||
@@ -119,7 +115,7 @@ public class DualStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
return storageRecord.getValue();
|
||||
}
|
||||
|
||||
//get from not commit cp info
|
||||
// get from not commit cp info
|
||||
List<Long> checkpointIds = new ArrayList<>(middleStore.keySet());
|
||||
Collections.sort(checkpointIds);
|
||||
for (int i = checkpointIds.size() - 1; i >= 0; i--) {
|
||||
|
||||
+1
-4
@@ -28,9 +28,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
/**
|
||||
* This class define the multi-version store strategy, which leverages external storage's mvcc.
|
||||
*/
|
||||
/** This class define the multi-version store strategy, which leverages external storage's mvcc. */
|
||||
public class MVStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
|
||||
public MVStateStoreManager(KeyValueStore<String, Map<Long, byte[]>> kvStore) {
|
||||
@@ -153,5 +151,4 @@ public class MVStateStoreManager<V> extends AbstractStateStoreManager<V> {
|
||||
this.middleStore.remove(commitBatchId);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+7
-13
@@ -27,22 +27,20 @@ import java.util.Map;
|
||||
|
||||
/**
|
||||
* This class support ITransactionState.
|
||||
* <p>
|
||||
* Based on the given StorageMode, different implementation instance of the AbstractStateStrategy
|
||||
*
|
||||
* <p>Based on the given StorageMode, different implementation instance of the AbstractStateStrategy
|
||||
* class will be created. All method calls will be delegated to the strategy instance.
|
||||
*/
|
||||
|
||||
public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
|
||||
|
||||
protected final AbstractStateStoreManager<V> stateStrategy;
|
||||
private final AbstractKeyStateBackend keyStateBackend;
|
||||
|
||||
public StateStoreManagerProxy(
|
||||
AbstractKeyStateBackend keyStateBackend,
|
||||
AbstractStateDescriptor stateDescriptor) {
|
||||
AbstractKeyStateBackend keyStateBackend, AbstractStateDescriptor stateDescriptor) {
|
||||
this.keyStateBackend = keyStateBackend;
|
||||
KeyValueStore<String, Map<Long, byte[]>> backStorage = keyStateBackend
|
||||
.getBackStorage(stateDescriptor);
|
||||
KeyValueStore<String, Map<Long, byte[]>> backStorage =
|
||||
keyStateBackend.getBackStorage(stateDescriptor);
|
||||
StateStrategy stateStrategy = keyStateBackend.getStateStrategy();
|
||||
switch (stateStrategy) {
|
||||
case DUAL_VERSION:
|
||||
@@ -65,17 +63,13 @@ public abstract class StateStoreManagerProxy<V> implements StateStoreManager {
|
||||
this.stateStrategy.finish(checkpointId);
|
||||
}
|
||||
|
||||
/**
|
||||
* The commit can be used in another thread to reach async state commit.
|
||||
*/
|
||||
/** The commit can be used in another thread to reach async state commit. */
|
||||
@Override
|
||||
public void commit(long checkpointId) {
|
||||
this.stateStrategy.commit(checkpointId);
|
||||
}
|
||||
|
||||
/**
|
||||
* The ackCommit must be called after commit in the same thread.
|
||||
*/
|
||||
/** The ackCommit must be called after commit in the same thread. */
|
||||
@Override
|
||||
public void ackCommit(long checkpointId, long timeStamp) {
|
||||
this.stateStrategy.ackCommit(checkpointId);
|
||||
|
||||
+7
-11
@@ -37,8 +37,8 @@ public class KeyStateBackendTest {
|
||||
|
||||
public void testGetValueState() {
|
||||
keyStateBackend.setCheckpointId(1L);
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("value", String.class, null);
|
||||
ValueStateDescriptor<String> valueStateDescriptor =
|
||||
ValueStateDescriptor.build("value", String.class, null);
|
||||
valueStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
ValueState<String> valueState = keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -114,13 +114,12 @@ public class KeyStateBackendTest {
|
||||
Assert.assertEquals(valueState.get(), "jack");
|
||||
valueState.setCurrentKey("9");
|
||||
Assert.assertNull(valueState.get());
|
||||
|
||||
}
|
||||
|
||||
public void testGetListState() {
|
||||
keyStateBackend.setCheckpointId(1l);
|
||||
ListStateDescriptor<String> listStateDescriptor = ListStateDescriptor
|
||||
.build("list", String.class);
|
||||
ListStateDescriptor<String> listStateDescriptor =
|
||||
ListStateDescriptor.build("list", String.class);
|
||||
listStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
ListState<String> listState = keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -207,9 +206,8 @@ public class KeyStateBackendTest {
|
||||
|
||||
public void testGetMapState() {
|
||||
keyStateBackend.setCheckpointId(1l);
|
||||
MapStateDescriptor<String, String> mapStateDescriptor = MapStateDescriptor
|
||||
.build("map", String.class,
|
||||
String.class);
|
||||
MapStateDescriptor<String, String> mapStateDescriptor =
|
||||
MapStateDescriptor.build("map", String.class, String.class);
|
||||
mapStateDescriptor.setTableName("kepler_hlg_ut");
|
||||
MapState<String, String> mapState = keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
@@ -223,7 +221,7 @@ public class KeyStateBackendTest {
|
||||
Assert.assertEquals(mapState.get("hello3"), null);
|
||||
|
||||
mapState.setCurrentKey("2");
|
||||
//Assert.assertEquals(mapState.iterator(), (new HashMap()));
|
||||
// Assert.assertEquals(mapState.iterator(), (new HashMap()));
|
||||
|
||||
mapState.setCurrentKey("2");
|
||||
mapState.put("eagle", "eagle-1");
|
||||
@@ -303,7 +301,6 @@ public class KeyStateBackendTest {
|
||||
Assert.assertNull(mapState.get("6666"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMem() {
|
||||
stateBackend = StateBackendBuilder.buildStateBackend(new HashMap<>());
|
||||
@@ -312,5 +309,4 @@ public class KeyStateBackendTest {
|
||||
testGetListState();
|
||||
testGetMapState();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -56,5 +56,4 @@ public class DefaultKeyMapStoreSerializationTest {
|
||||
byte[] result = this.defaultKMapStoreSerDe.serializeUValue(value);
|
||||
Assert.assertEquals(value, this.defaultKMapStoreSerDe.deserializeUValue(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-1
@@ -58,7 +58,6 @@ public class MemoryKeyMapStoreTest {
|
||||
Assert.assertNotEquals(IKeyMapStore.get("hello"), map);
|
||||
Assert.assertEquals(IKeyMapStore.get("hello"), map2);
|
||||
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
-3
@@ -20,12 +20,10 @@ package io.ray.streaming.state.keystate;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class KeyGroupAssignmentTest {
|
||||
|
||||
|
||||
@Test
|
||||
public void testComputeKeyGroupRangeForOperatorIndex() throws Exception {
|
||||
KeyGroup keyGroup = KeyGroupAssignment.getKeyGroup(4096, 1, 0);
|
||||
@@ -53,5 +51,4 @@ public class KeyGroupAssignmentTest {
|
||||
assertEquals(keyGroup3.getEndIndex(), 4095);
|
||||
assertEquals(keyGroup3.size(), 1365);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-3
@@ -26,8 +26,8 @@ public class ListStateDescriptorTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
ListStateDescriptor<Integer> descriptor = ListStateDescriptor
|
||||
.build("lsdTest", Integer.class, true);
|
||||
ListStateDescriptor<Integer> descriptor =
|
||||
ListStateDescriptor.build("lsdTest", Integer.class, true);
|
||||
descriptor.setTableName("table");
|
||||
Assert.assertTrue(descriptor.isOperatorList());
|
||||
|
||||
@@ -37,7 +37,8 @@ public class ListStateDescriptorTest {
|
||||
Assert.assertEquals(descriptor.getPartitionNumber(), 3);
|
||||
Assert.assertEquals(descriptor.getIndex(), 0);
|
||||
|
||||
Assert.assertEquals(descriptor.getIdentify(),
|
||||
Assert.assertEquals(
|
||||
descriptor.getIdentify(),
|
||||
"lsdTest" + ConfigKey.DELIMITER + "3" + ConfigKey.DELIMITER + "0");
|
||||
}
|
||||
}
|
||||
|
||||
+2
-4
@@ -25,13 +25,11 @@ public class MapStateDescriptorTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
|
||||
.build("msdTest", String.class,
|
||||
Integer.class);
|
||||
MapStateDescriptor<String, Integer> descriptor =
|
||||
MapStateDescriptor.build("msdTest", String.class, Integer.class);
|
||||
|
||||
descriptor.setTableName("table");
|
||||
Assert.assertEquals(descriptor.getTableName(), "table");
|
||||
Assert.assertEquals(descriptor.getName(), "msdTest");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,8 +25,8 @@ public class ValueStateDescriptorTest {
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
ValueStateDescriptor<Integer> descriptor = ValueStateDescriptor
|
||||
.build("vsdTest", Integer.class, 0);
|
||||
ValueStateDescriptor<Integer> descriptor =
|
||||
ValueStateDescriptor.build("vsdTest", Integer.class, 0);
|
||||
Assert.assertEquals(descriptor.getDefaultValue().intValue(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-5
@@ -36,10 +36,10 @@ public class ListStateImplTest {
|
||||
|
||||
@BeforeClass
|
||||
public void setUp() throws Exception {
|
||||
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
|
||||
new MemoryStateBackend(new HashMap<>()));
|
||||
ListStateDescriptor<Integer> descriptor = ListStateDescriptor
|
||||
.build("ListStateImplTest", Integer.class);
|
||||
keyStateBackend =
|
||||
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
|
||||
ListStateDescriptor<Integer> descriptor =
|
||||
ListStateDescriptor.build("ListStateImplTest", Integer.class);
|
||||
descriptor.setTableName("table");
|
||||
|
||||
listState = (ListStateImpl<Integer>) keyStateBackend.getListState(descriptor);
|
||||
@@ -72,7 +72,6 @@ public class ListStateImplTest {
|
||||
Assert.assertEquals(listState.get(), Arrays.asList(5, 6));
|
||||
}
|
||||
|
||||
|
||||
@Test(dependsOnMethods = {"testAddGet"})
|
||||
public void testUpdate() throws Exception {
|
||||
Assert.assertEquals(listState.get(), Arrays.asList(5, 6));
|
||||
|
||||
+4
-6
@@ -36,11 +36,10 @@ public class MapStateImplTest {
|
||||
|
||||
@BeforeClass
|
||||
public void setUp() throws Exception {
|
||||
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
|
||||
new MemoryStateBackend(new HashMap<>()));
|
||||
MapStateDescriptor<Integer, String> descriptor = MapStateDescriptor
|
||||
.build("MapStateImplTest", Integer.class,
|
||||
String.class);
|
||||
keyStateBackend =
|
||||
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
|
||||
MapStateDescriptor<Integer, String> descriptor =
|
||||
MapStateDescriptor.build("MapStateImplTest", Integer.class, String.class);
|
||||
descriptor.setTableName("table");
|
||||
mapState = (MapStateImpl<Integer, String>) keyStateBackend.getMapState(descriptor);
|
||||
}
|
||||
@@ -88,7 +87,6 @@ public class MapStateImplTest {
|
||||
Assert.assertEquals(mapState.get().size(), 0);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testFailover() throws Exception {
|
||||
keyStateBackend.setContext(1L, 1);
|
||||
|
||||
+3
-3
@@ -41,9 +41,9 @@ public class OperatorStateImplTest {
|
||||
|
||||
operatorStateBackend = new OperatorStateBackend(new MemoryStateBackend(config));
|
||||
|
||||
descriptor = ListStateDescriptor
|
||||
.build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class,
|
||||
true);
|
||||
descriptor =
|
||||
ListStateDescriptor.build(
|
||||
"OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, true);
|
||||
descriptor.setPartitionNumber(1);
|
||||
descriptor.setIndex(0);
|
||||
descriptor.setTableName(table_name);
|
||||
|
||||
+4
-5
@@ -34,11 +34,10 @@ public class ValueStateImplTest {
|
||||
|
||||
@BeforeClass
|
||||
public void setUp() throws Exception {
|
||||
keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2),
|
||||
new MemoryStateBackend(new HashMap<>()));
|
||||
ValueStateDescriptor<String> descriptor = ValueStateDescriptor
|
||||
.build("ValueStateImplTest", String.class,
|
||||
"hello");
|
||||
keyStateBackend =
|
||||
new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>()));
|
||||
ValueStateDescriptor<String> descriptor =
|
||||
ValueStateDescriptor.build("ValueStateImplTest", String.class, "hello");
|
||||
descriptor.setTableName("table");
|
||||
|
||||
valueState = (ValueStateImpl<String>) keyStateBackend.getValueState(descriptor);
|
||||
|
||||
+2
-3
@@ -31,8 +31,8 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest {
|
||||
|
||||
@BeforeClass
|
||||
public void setUp() {
|
||||
MapStateDescriptor<String, Integer> descriptor = MapStateDescriptor
|
||||
.build("map", String.class, Integer.class);
|
||||
MapStateDescriptor<String, Integer> descriptor =
|
||||
MapStateDescriptor.build("map", String.class, Integer.class);
|
||||
descriptor.setTableName("tableName");
|
||||
keyStateBackend.setContext(1L, "key");
|
||||
facade = new MapStateStoreManagerProxy<>(keyStateBackend, descriptor);
|
||||
@@ -52,5 +52,4 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest {
|
||||
facade.put("key2", map);
|
||||
Assert.assertEquals(facade.get("key2"), map);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-2
@@ -25,6 +25,7 @@ import java.util.HashMap;
|
||||
|
||||
public class StateStoreManagerTest {
|
||||
|
||||
protected KeyStateBackend keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 1),
|
||||
new MemoryStateBackend(new HashMap<String, String>()));
|
||||
protected KeyStateBackend keyStateBackend =
|
||||
new KeyStateBackend(
|
||||
1, new KeyGroup(1, 1), new MemoryStateBackend(new HashMap<String, String>()));
|
||||
}
|
||||
|
||||
+2
-2
@@ -30,8 +30,8 @@ public class ValueStateStoreManagerTest extends StateStoreManagerTest {
|
||||
|
||||
@BeforeClass
|
||||
public void setUp() {
|
||||
ValueStateDescriptor<Integer> descriptor = ValueStateDescriptor
|
||||
.build("value", Integer.class, 0);
|
||||
ValueStateDescriptor<Integer> descriptor =
|
||||
ValueStateDescriptor.build("value", Integer.class, 0);
|
||||
descriptor.setTableName("tableName");
|
||||
keyStateBackend.setContext(1L, "key");
|
||||
proxy = new ValueStateStoreManagerProxy<>(keyStateBackend, descriptor);
|
||||
|
||||
+13
-18
@@ -54,9 +54,8 @@ public class DualStateStrategyTest {
|
||||
}
|
||||
|
||||
public void caseKV() {
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("VALUE-" + currentTime,
|
||||
String.class, defaultValue);
|
||||
ValueStateDescriptor<String> valueStateDescriptor =
|
||||
ValueStateDescriptor.build("VALUE-" + currentTime, String.class, defaultValue);
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -116,7 +115,7 @@ public class DualStateStrategyTest {
|
||||
this.keyStateBackend.commit(2);
|
||||
this.keyStateBackend.ackCommit(2, 2);
|
||||
|
||||
//do rollback, all memory data is deleted.
|
||||
// do rollback, all memory data is deleted.
|
||||
this.keyStateBackend.rollBack(1);
|
||||
this.keyStateBackend.setCheckpointId(1);
|
||||
state.setCurrentKey(("1"));
|
||||
@@ -159,9 +158,8 @@ public class DualStateStrategyTest {
|
||||
}
|
||||
|
||||
public void caseKVGap() {
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("value2-" + currentTime,
|
||||
String.class, defaultValue);
|
||||
ValueStateDescriptor<String> valueStateDescriptor =
|
||||
ValueStateDescriptor.build("value2-" + currentTime, String.class, defaultValue);
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -205,13 +203,11 @@ public class DualStateStrategyTest {
|
||||
this.keyStateBackend.setCheckpointId(11);
|
||||
this.keyStateBackend.rollBack(11);
|
||||
Assert.assertEquals(state.get(), "info");
|
||||
|
||||
}
|
||||
|
||||
public void caseKList() {
|
||||
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
|
||||
.build("LIST-" + currentTime,
|
||||
Integer.class);
|
||||
ListStateDescriptor<Integer> listStateDescriptor =
|
||||
ListStateDescriptor.build("LIST-" + currentTime, Integer.class);
|
||||
listStateDescriptor.setTableName(table);
|
||||
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -272,7 +268,7 @@ public class DualStateStrategyTest {
|
||||
this.keyStateBackend.commit(2);
|
||||
this.keyStateBackend.ackCommit(2, 2);
|
||||
|
||||
//do rollback, all memory data is deleted.
|
||||
// do rollback, all memory data is deleted.
|
||||
this.keyStateBackend.rollBack(1);
|
||||
this.keyStateBackend.setCheckpointId(1);
|
||||
state.setCurrentKey(("1"));
|
||||
@@ -312,9 +308,8 @@ public class DualStateStrategyTest {
|
||||
}
|
||||
|
||||
public void caseKMap() {
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
|
||||
.build("MAP-" + currentTime,
|
||||
Integer.class, Integer.class);
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor =
|
||||
MapStateDescriptor.build("MAP-" + currentTime, Integer.class, Integer.class);
|
||||
mapStateDescriptor.setTableName(table);
|
||||
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
@@ -378,7 +373,7 @@ public class DualStateStrategyTest {
|
||||
this.keyStateBackend.commit(2);
|
||||
this.keyStateBackend.ackCommit(2, 2);
|
||||
|
||||
//do rollback, memory data is deleted.
|
||||
// do rollback, memory data is deleted.
|
||||
this.keyStateBackend.rollBack(1);
|
||||
this.keyStateBackend.setCheckpointId(1);
|
||||
state.setCurrentKey(("1"));
|
||||
@@ -428,8 +423,8 @@ public class DualStateStrategyTest {
|
||||
@Test
|
||||
public void testMem() {
|
||||
config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name());
|
||||
this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3),
|
||||
StateBackendBuilder.buildStateBackend(config));
|
||||
this.keyStateBackend =
|
||||
new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config));
|
||||
caseKV();
|
||||
caseKVGap();
|
||||
caseKList();
|
||||
|
||||
+8
-13
@@ -52,9 +52,8 @@ public class MVStateStrategyTest {
|
||||
|
||||
public void caseKV() {
|
||||
|
||||
ValueStateDescriptor<String> valueStateDescriptor = ValueStateDescriptor
|
||||
.build("mvint-" + currentTime,
|
||||
String.class, "");
|
||||
ValueStateDescriptor<String> valueStateDescriptor =
|
||||
ValueStateDescriptor.build("mvint-" + currentTime, String.class, "");
|
||||
valueStateDescriptor.setTableName(table);
|
||||
ValueState<String> state = this.keyStateBackend.getValueState(valueStateDescriptor);
|
||||
|
||||
@@ -170,9 +169,8 @@ public class MVStateStrategyTest {
|
||||
}
|
||||
|
||||
public void caseKList() {
|
||||
ListStateDescriptor<Integer> listStateDescriptor = ListStateDescriptor
|
||||
.build("mvlist-" + currentTime,
|
||||
Integer.class);
|
||||
ListStateDescriptor<Integer> listStateDescriptor =
|
||||
ListStateDescriptor.build("mvlist-" + currentTime, Integer.class);
|
||||
listStateDescriptor.setTableName(table);
|
||||
ListState<Integer> state = this.keyStateBackend.getListState(listStateDescriptor);
|
||||
|
||||
@@ -275,9 +273,8 @@ public class MVStateStrategyTest {
|
||||
}
|
||||
|
||||
public void caseKMap() {
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor = MapStateDescriptor
|
||||
.build("mvmap-" + currentTime,
|
||||
Integer.class, Integer.class);
|
||||
MapStateDescriptor<Integer, Integer> mapStateDescriptor =
|
||||
MapStateDescriptor.build("mvmap-" + currentTime, Integer.class, Integer.class);
|
||||
mapStateDescriptor.setTableName(table);
|
||||
MapState<Integer, Integer> state = this.keyStateBackend.getMapState(mapStateDescriptor);
|
||||
|
||||
@@ -394,15 +391,13 @@ public class MVStateStrategyTest {
|
||||
Assert.assertNull(state.get(7));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMemMV() {
|
||||
config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name());
|
||||
this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3),
|
||||
StateBackendBuilder.buildStateBackend(config));
|
||||
this.keyStateBackend =
|
||||
new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config));
|
||||
caseKV();
|
||||
caseKList();
|
||||
caseKMap();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user