diff --git a/doublearray/doublearray-tests.ts b/doublearray/doublearray-tests.ts new file mode 100644 index 000000000..e54a58fd0 --- /dev/null +++ b/doublearray/doublearray-tests.ts @@ -0,0 +1,32 @@ +/// + +// https://github.com/takuyaa/doublearray/blob/master/README.md +var words = [ + { k: 'a', v: 1 }, + { k: 'abc', v: 2 }, + { k: '奈良', v: 3 }, + { k: '奈良先端', v: 4 }, + { k: '奈良先端科学技術大学院大学', v: 5 } +]; + +var trie = doublearray.builder().build(words); + +var trie = doublearray + .builder() + .append('a', 1) + .append('abc', 2) + .append('奈良', 3) + .append('奈良先端', 4) + .append('奈良先端科学技術大学院大学', 5) + .build(); + +trie.contain('a'); + +trie.lookup('abc'); + +trie.commonPrefixSearch('奈良先端科学技術大学院大学'); + +var base_buffer: Int32Array = trie.bc.getBaseBuffer(); +var check_buffer: Int32Array = trie.bc.getCheckBuffer(); + +var loaded_trie = doublearray.load(base_buffer, check_buffer); \ No newline at end of file diff --git a/doublearray/doublearray.d.ts b/doublearray/doublearray.d.ts new file mode 100644 index 000000000..b34aa4cb8 --- /dev/null +++ b/doublearray/doublearray.d.ts @@ -0,0 +1,99 @@ +// Type definitions for doublearray +// Project: https://github.com/takuyaa/doublearray +// Definitions by: MIZUSHIMA Junki +// Definitions: https://github.com/borisyankov/DefinitelyTyped + +declare module doublearray { + interface KeyValue { + k: string; + v: number; + } + + interface BaseAndCheck { + getBaseBuffer(): any; // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array + getCheckBuffer(): any; // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array + loadBaseBuffer(base_buffer: Int8Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Int16Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Int32Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint8Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint16Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint32Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int8Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int16Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int32Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint8Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint16Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint32Array): BaseAndCheck; + size(): number; + getBase(): number; + getCheck(): number; + setBase(index: number, base_value: number): void; + setCheck(index: number, check_value: number): void; + setFirstUnusedNode(index: number): void; + getFirstUnusedNode(): number; + shrink(): void; + calc(): {all: number; unused: number; efficiency: number}; + dump(): string; + } + + interface DoubleArrayBuilder { + bc: BaseAndCheck; + keys: KeyValue[]; + append(key: string, record: number): DoubleArrayBuilder; + build(keys?: KeyValue[], sorted?: boolean): DoubleArray; + getChildrenInfo(position: number, start: number, length: number): Int32Array; + setBC(parent_id: number, children_info: Int32Array, _base: number): void; + findAllocatableBase(children_info: Int32Array): number; + isUnusedNode(index: number): boolean; + } + + interface DoubleArray { + bc: BaseAndCheck; + contain(key: string): boolean; + lookup(key: string): number; + commonPrefixSearch(key: string): KeyValue; + traverse(parent: number, code: number): number; + size(): number; + calc(): {all: number; unused: number; efficiency: number}; + dump(): string; + } + + export function builder(initial_size?: number): DoubleArrayBuilder; + // TODO: Replace to union types in the future. + export function load(base_buffer: Int8Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint32Array): DoubleArray; +} \ No newline at end of file diff --git a/kuromoji/kuromoji-tests.ts b/kuromoji/kuromoji-tests.ts new file mode 100644 index 000000000..0355a6501 --- /dev/null +++ b/kuromoji/kuromoji-tests.ts @@ -0,0 +1,33 @@ +/// + +// From https://github.com/takuyaa/kuromoji.js/blob/master/README.md#usage +kuromoji.builder({ dicPath: "/url/to/dictionary/dir/" }).build(function (err, tokenizer) { + var path = tokenizer.tokenize("すもももももももものうち"); + var num_tmp: number; + var str_tmp: string; + path.forEach((token)=>{ + num_tmp = token.word_id; + str_tmp = token.word_type; + num_tmp = token.word_position; + str_tmp = token.surface_form; + str_tmp = token.pos; + str_tmp = token.pos_detail_1; + str_tmp = token.pos_detail_2; + str_tmp = token.pos_detail_3; + str_tmp = token.conjugated_type; + str_tmp = token.conjugated_form; + str_tmp = token.basic_form; + str_tmp = token.reading; + str_tmp = token.pronunciation; + }); +}); + +// From https://github.com/takuyaa/kuromoji.js/blob/master/test/resource/minimum-dic/minimum.csv +var minimum_dict = [ + "すもも,1285,1285,7546,名詞,一般,*,*,*,*,すもも,スモモ,スモモ", + "もも,1285,1285,7219,名詞,一般,*,*,*,*,もも,モモ,モモ" +].join('\n'); + +var builder = kuromoji.dictionaryBuilder(); +builder = builder.addTokenInfoDictionary(minimum_dict); +var dict = builder.build(); \ No newline at end of file diff --git a/kuromoji/kuromoji.d.ts b/kuromoji/kuromoji.d.ts new file mode 100644 index 000000000..ed1cd9b80 --- /dev/null +++ b/kuromoji/kuromoji.d.ts @@ -0,0 +1,160 @@ +// Type definitions for kuromoji.js +// Project: https://github.com/takuyaa/kuromoji.js +// Definitions by: MIZUSHIMA Junki +// Definitions: https://github.com/borisyankov/DefinitelyTyped + +/// + +declare module kuromoji { + + // dict/ConnectionCosts.js + interface ConnectionCosts { + buffer: Int16Array; + put(forward_id: number, backward_id: number, cost: number): void; + get(forward_id: number, backward_id: number): number; + loadConnectionCosts(connection_costs_buffer: Int16Array): void; + } + + // dict/DynamicDictionaries.js + interface DynamicDictionaries { + trie: doublearray.DoubleArray; + token_info_dictionary: TokenInfoDictionary; + connection_costs: ConnectionCosts; + unknown_dictionary: UnknownDictionary; + loadTrie(base_buffer: Int32Array, check_buffer: Int32Array): DynamicDictionaries; + } + + // dict/TokenInfoDictionary.js + interface TokenInfoDictionary { + buildDictionary(entries: any[][]): {[word_id: number]: string}; + put(left_id: number, right_id: number, word_cost: number, surface_form: string, feature: string): number; + addMapping(source: number, target: number): void; + targetMapToBuffer(): Uint8Array; + loadDictionary(array_buffer: Uint8Array): TokenInfoDictionary; + loadPosVector(array_buffer: Uint8Array): TokenInfoDictionary; + loadTargetMap(array_buffer: Uint8Array): TokenInfoDictionary; + getFeatures(token_info_id_str: string): string; + } + + // dict/UnknownDictionary.js + interface UnknownDictionary extends TokenInfoDictionary { + } + + // util/ByteBuffer.js + interface ByteBuffer { + buffer: Uint8Array; + position: number; + size(): number; + reallocate(): void; + shrink(): Uint8Array; + put(b: number): void; + get(index: number): number; + putShort(num: number): void; + getShort(index: number): number; + putInt(num: number): void; + getInt(index: number): number; + readInt(): number; + putString(str: string): void; + getString(index: number): string; + } + + // util/DictionaryBuilder.js + interface DictionaryBuilder { + tid_entries: string[]; + unk_entries: string[]; + addTokenInfoDictionary(text: string): DictionaryBuilder; + costMatrix(matrix_text: string): DictionaryBuilder; + charDef(char_text: string): DictionaryBuilder; + unkDef(text: string): DictionaryBuilder; + build(): DynamicDictionaries; + buildTokenInfoDictionary(): {trie: doublearray.DoubleArray; token_info_dictionary: TokenInfoDictionary}; + buildUnknownDictionary(): UnknownDictionary; + buildConnectionCosts(): ConnectionCosts; + buildDoubleArray(): doublearray.DoubleArray; + } + + // util/IpadicFormatter.js + interface Formatter { + formatEntry(word_id: number, position: number, type: string, features: string[]): T; + formatUnknownEntry(word_id: number, position: number, type: string, features: string[]): T; + } + interface IpadicFormatter extends Formatter { + } + export interface IpadicFeatures { + word_id: number; + word_type: string; + word_position: number; + surface_form: string; + pos: string; + pos_detail_1: string; + pos_detail_2: string; + pos_detail_3: string; + conjugated_type: string; + conjugated_form: string; + basic_form: string; + reading?: string; + pronunciation?: string; + } + + // viterbi/ViterbiBuilder.js + interface ViterbiBuilder { + trie: doublearray.DoubleArray; + token_info_dictionary: TokenInfoDictionary; + unknown_dictionary: UnknownDictionary; + build(sentence_str: string): ViterbiLattice; + } + + // viterbi/ViterbiLattice.js + interface ViterbiLattice { + append(node: ViterbiNode): void; + appendEos(): void; + } + + // viterbi/ViterbiNode.js + interface ViterbiNode { + name: string; + cost: number; + start_pos: number; + length: number; + left_id: number; + right_id: number; + prev: ViterbiNode; + surface_form: string; + shortest_cost: number; + type: string; + } + + // viterbi/ViterbiSearcher.js + interface ViterbiSearcher { + connection_costs: ConnectionCosts; + search(lattice: ViterbiLattice): ViterbiNode[]; + forward(lattice: ViterbiLattice): ViterbiLattice; + backward(lattice: ViterbiLattice): ViterbiNode[]; + } + + // Tokenizer.js + interface TokenizerStatic { + splitByPunctuation(input: string): string[]; + } + interface Tokenizer { + token_info_dictionary: TokenInfoDictionary; + unknown_dictionary: UnknownDictionary; + viterbi_builder: ViterbiBuilder; + viterbi_searcher: ViterbiSearcher; + formatter: Formatter; + tokenize(text: string): T[]; + getLattice(text: string): ViterbiLattice; + } + + // TokenizerBuilder.js + interface TokenizerBuilder { + build(callback: (err: Error, tokenizer: Tokenizer) => void): void; + } + interface TokenizerBuilderOption { + dicPath?: string; + } + + // kuromoji.js + export function builder(option: TokenizerBuilderOption): TokenizerBuilder; + export function dictionaryBuilder(): DictionaryBuilder; +} \ No newline at end of file