From fb61e6dd471914ec14d8615b138ea75ee62e2443 Mon Sep 17 00:00:00 2001 From: mzsm Date: Fri, 26 Dec 2014 04:18:29 +0900 Subject: [PATCH 1/3] add double array --- doublearray/doublearray-tests.ts | 32 +++++++++++ doublearray/doublearray.d.ts | 99 ++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 doublearray/doublearray-tests.ts create mode 100644 doublearray/doublearray.d.ts diff --git a/doublearray/doublearray-tests.ts b/doublearray/doublearray-tests.ts new file mode 100644 index 000000000..e54a58fd0 --- /dev/null +++ b/doublearray/doublearray-tests.ts @@ -0,0 +1,32 @@ +/// + +// https://github.com/takuyaa/doublearray/blob/master/README.md +var words = [ + { k: 'a', v: 1 }, + { k: 'abc', v: 2 }, + { k: '奈良', v: 3 }, + { k: '奈良先端', v: 4 }, + { k: '奈良先端科学技術大学院大学', v: 5 } +]; + +var trie = doublearray.builder().build(words); + +var trie = doublearray + .builder() + .append('a', 1) + .append('abc', 2) + .append('奈良', 3) + .append('奈良先端', 4) + .append('奈良先端科学技術大学院大学', 5) + .build(); + +trie.contain('a'); + +trie.lookup('abc'); + +trie.commonPrefixSearch('奈良先端科学技術大学院大学'); + +var base_buffer: Int32Array = trie.bc.getBaseBuffer(); +var check_buffer: Int32Array = trie.bc.getCheckBuffer(); + +var loaded_trie = doublearray.load(base_buffer, check_buffer); \ No newline at end of file diff --git a/doublearray/doublearray.d.ts b/doublearray/doublearray.d.ts new file mode 100644 index 000000000..b34aa4cb8 --- /dev/null +++ b/doublearray/doublearray.d.ts @@ -0,0 +1,99 @@ +// Type definitions for doublearray +// Project: https://github.com/takuyaa/doublearray +// Definitions by: MIZUSHIMA Junki +// Definitions: https://github.com/borisyankov/DefinitelyTyped + +declare module doublearray { + interface KeyValue { + k: string; + v: number; + } + + interface BaseAndCheck { + getBaseBuffer(): any; // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array + getCheckBuffer(): any; // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array + loadBaseBuffer(base_buffer: Int8Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Int16Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Int32Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint8Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint16Array): BaseAndCheck; + loadBaseBuffer(base_buffer: Uint32Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int8Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int16Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Int32Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint8Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint16Array): BaseAndCheck; + loadCheckBuffer(check_buffer: Uint32Array): BaseAndCheck; + size(): number; + getBase(): number; + getCheck(): number; + setBase(index: number, base_value: number): void; + setCheck(index: number, check_value: number): void; + setFirstUnusedNode(index: number): void; + getFirstUnusedNode(): number; + shrink(): void; + calc(): {all: number; unused: number; efficiency: number}; + dump(): string; + } + + interface DoubleArrayBuilder { + bc: BaseAndCheck; + keys: KeyValue[]; + append(key: string, record: number): DoubleArrayBuilder; + build(keys?: KeyValue[], sorted?: boolean): DoubleArray; + getChildrenInfo(position: number, start: number, length: number): Int32Array; + setBC(parent_id: number, children_info: Int32Array, _base: number): void; + findAllocatableBase(children_info: Int32Array): number; + isUnusedNode(index: number): boolean; + } + + interface DoubleArray { + bc: BaseAndCheck; + contain(key: string): boolean; + lookup(key: string): number; + commonPrefixSearch(key: string): KeyValue; + traverse(parent: number, code: number): number; + size(): number; + calc(): {all: number; unused: number; efficiency: number}; + dump(): string; + } + + export function builder(initial_size?: number): DoubleArrayBuilder; + // TODO: Replace to union types in the future. + export function load(base_buffer: Int8Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int8Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int16Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Int32Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint8Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint16Array, check_buffer: Uint32Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int8Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int16Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Int32Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint8Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint16Array): DoubleArray; + export function load(base_buffer: Uint32Array, check_buffer: Uint32Array): DoubleArray; +} \ No newline at end of file From f4007ed324a71ef6e3f87d0132dc584bec1d61a6 Mon Sep 17 00:00:00 2001 From: mzsm Date: Thu, 25 Dec 2014 19:55:28 +0900 Subject: [PATCH 2/3] Add kuromoji.js --- kuromoji/kuromoji-tests.ts | 9 ++++ kuromoji/kuromoji.d.ts | 96 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 kuromoji/kuromoji-tests.ts create mode 100644 kuromoji/kuromoji.d.ts diff --git a/kuromoji/kuromoji-tests.ts b/kuromoji/kuromoji-tests.ts new file mode 100644 index 000000000..d19e7f140 --- /dev/null +++ b/kuromoji/kuromoji-tests.ts @@ -0,0 +1,9 @@ +/// + +kuromoji.builder({ dicPath: "/url/to/dictionary/dir/" }).build(function (err, tokenizer) { + var path = tokenizer.tokenize("すもももももももものうち"); + path.forEach((token)=>{ + console.log(token.word_id); + console.log(token.surface_form); + }); +}); \ No newline at end of file diff --git a/kuromoji/kuromoji.d.ts b/kuromoji/kuromoji.d.ts new file mode 100644 index 000000000..7ee5ca92a --- /dev/null +++ b/kuromoji/kuromoji.d.ts @@ -0,0 +1,96 @@ +// Type definitions for kuromoji.js +// Project: https://github.com/takuyaa/kuromoji.js +// Definitions by: MIZUSHIMA Junki +// Definitions: https://github.com/borisyankov/DefinitelyTyped + +declare module kuromoji { + + interface TokenizerBuilderOption { + dicPath?: string; + } + + interface TokenizerBuilder { + build(callback: (err: Error, tokenizer: Tokenizer) => void): void; + } + + interface Tokenizer { + token_info_dictionary: any; + unknown_dictionary: any; + viterbi_builder: ViterbiBuilder; + viterbi_searcher: ViterbiSearcher; + formatter: T; + tokenize(text: string): T[]; + getLattice(text: string): ViterbiLattice; + } + + interface ViterbiBuilder { + trie: any; + token_info_dictionary: any; + unknown_dictionary: any; + build(sentence_str: string): ViterbiLattice; + + } + + interface ViterbiSearcher { + connection_costs: any; + search(lattice: ViterbiLattice): ViterbiNode[]; + forward(lattice: ViterbiLattice) + } + + interface ViterbiLattice { + append(node: ViterbiNode): void; + appendEos(): void; + } + + interface ViterbiNode { + name: string; + cost: number; + start_pos: number; + length: number; + left_id: number; + right_id: number; + prev: ViterbiNode; + surface_form: string; + shortest_cost: number; + type: string; + } + + interface IpadicFormatter { + formatEntry(word_id: number, position: number, type: string, features: string[]): IpadicFormat; + formatUnknownEntry(word_id: number, position: number, type: string, features: string[]): IpadicFormat; + } + + interface IpadicFormat { + word_id: number; + word_type: string; + word_position: number; + surface_form: number; + pos: string; + pos_detail_1: string; + pos_detail_2: string; + pos_detail_3: string; + conjugated_type: string; + conjugated_form: string; + basic_form: string; + reading?: string; + pronunciation?: string; + } + + interface DictionaryBuilder { + tid_entries: number[]; + unk_entries: number[]; + matrix_text: string; + char_text: string; + addTokenInfoDictionary(text: string): DictionaryBuilder; + costMatrix(matrix_text: string): DictionaryBuilder; + charDef(char_text: string): DictionaryBuilder; + unkDef(text: string): DictionaryBuilder; + build(): DynamicDictionaries; + } + + interface DynamicDictionaries { + } + + export function builder(option?: TokenizerBuilderOption): TokenizerBuilder; + export function dictionaryBuilder(): DictionaryBuilder; +} \ No newline at end of file From fe562facd348b3d75ffbb9ee111a2d2c208fee44 Mon Sep 17 00:00:00 2001 From: mzsm Date: Fri, 26 Dec 2014 06:36:53 +0900 Subject: [PATCH 3/3] update kuromoji.js --- kuromoji/kuromoji-tests.ts | 30 ++++++- kuromoji/kuromoji.d.ts | 170 +++++++++++++++++++++++++------------ 2 files changed, 144 insertions(+), 56 deletions(-) diff --git a/kuromoji/kuromoji-tests.ts b/kuromoji/kuromoji-tests.ts index d19e7f140..0355a6501 100644 --- a/kuromoji/kuromoji-tests.ts +++ b/kuromoji/kuromoji-tests.ts @@ -1,9 +1,33 @@ /// +// From https://github.com/takuyaa/kuromoji.js/blob/master/README.md#usage kuromoji.builder({ dicPath: "/url/to/dictionary/dir/" }).build(function (err, tokenizer) { var path = tokenizer.tokenize("すもももももももものうち"); + var num_tmp: number; + var str_tmp: string; path.forEach((token)=>{ - console.log(token.word_id); - console.log(token.surface_form); + num_tmp = token.word_id; + str_tmp = token.word_type; + num_tmp = token.word_position; + str_tmp = token.surface_form; + str_tmp = token.pos; + str_tmp = token.pos_detail_1; + str_tmp = token.pos_detail_2; + str_tmp = token.pos_detail_3; + str_tmp = token.conjugated_type; + str_tmp = token.conjugated_form; + str_tmp = token.basic_form; + str_tmp = token.reading; + str_tmp = token.pronunciation; }); -}); \ No newline at end of file +}); + +// From https://github.com/takuyaa/kuromoji.js/blob/master/test/resource/minimum-dic/minimum.csv +var minimum_dict = [ + "すもも,1285,1285,7546,名詞,一般,*,*,*,*,すもも,スモモ,スモモ", + "もも,1285,1285,7219,名詞,一般,*,*,*,*,もも,モモ,モモ" +].join('\n'); + +var builder = kuromoji.dictionaryBuilder(); +builder = builder.addTokenInfoDictionary(minimum_dict); +var dict = builder.build(); \ No newline at end of file diff --git a/kuromoji/kuromoji.d.ts b/kuromoji/kuromoji.d.ts index 7ee5ca92a..ed1cd9b80 100644 --- a/kuromoji/kuromoji.d.ts +++ b/kuromoji/kuromoji.d.ts @@ -3,45 +3,114 @@ // Definitions by: MIZUSHIMA Junki // Definitions: https://github.com/borisyankov/DefinitelyTyped +/// + declare module kuromoji { - interface TokenizerBuilderOption { - dicPath?: string; + // dict/ConnectionCosts.js + interface ConnectionCosts { + buffer: Int16Array; + put(forward_id: number, backward_id: number, cost: number): void; + get(forward_id: number, backward_id: number): number; + loadConnectionCosts(connection_costs_buffer: Int16Array): void; } - interface TokenizerBuilder { - build(callback: (err: Error, tokenizer: Tokenizer) => void): void; + // dict/DynamicDictionaries.js + interface DynamicDictionaries { + trie: doublearray.DoubleArray; + token_info_dictionary: TokenInfoDictionary; + connection_costs: ConnectionCosts; + unknown_dictionary: UnknownDictionary; + loadTrie(base_buffer: Int32Array, check_buffer: Int32Array): DynamicDictionaries; } - interface Tokenizer { - token_info_dictionary: any; - unknown_dictionary: any; - viterbi_builder: ViterbiBuilder; - viterbi_searcher: ViterbiSearcher; - formatter: T; - tokenize(text: string): T[]; - getLattice(text: string): ViterbiLattice; + // dict/TokenInfoDictionary.js + interface TokenInfoDictionary { + buildDictionary(entries: any[][]): {[word_id: number]: string}; + put(left_id: number, right_id: number, word_cost: number, surface_form: string, feature: string): number; + addMapping(source: number, target: number): void; + targetMapToBuffer(): Uint8Array; + loadDictionary(array_buffer: Uint8Array): TokenInfoDictionary; + loadPosVector(array_buffer: Uint8Array): TokenInfoDictionary; + loadTargetMap(array_buffer: Uint8Array): TokenInfoDictionary; + getFeatures(token_info_id_str: string): string; } + // dict/UnknownDictionary.js + interface UnknownDictionary extends TokenInfoDictionary { + } + + // util/ByteBuffer.js + interface ByteBuffer { + buffer: Uint8Array; + position: number; + size(): number; + reallocate(): void; + shrink(): Uint8Array; + put(b: number): void; + get(index: number): number; + putShort(num: number): void; + getShort(index: number): number; + putInt(num: number): void; + getInt(index: number): number; + readInt(): number; + putString(str: string): void; + getString(index: number): string; + } + + // util/DictionaryBuilder.js + interface DictionaryBuilder { + tid_entries: string[]; + unk_entries: string[]; + addTokenInfoDictionary(text: string): DictionaryBuilder; + costMatrix(matrix_text: string): DictionaryBuilder; + charDef(char_text: string): DictionaryBuilder; + unkDef(text: string): DictionaryBuilder; + build(): DynamicDictionaries; + buildTokenInfoDictionary(): {trie: doublearray.DoubleArray; token_info_dictionary: TokenInfoDictionary}; + buildUnknownDictionary(): UnknownDictionary; + buildConnectionCosts(): ConnectionCosts; + buildDoubleArray(): doublearray.DoubleArray; + } + + // util/IpadicFormatter.js + interface Formatter { + formatEntry(word_id: number, position: number, type: string, features: string[]): T; + formatUnknownEntry(word_id: number, position: number, type: string, features: string[]): T; + } + interface IpadicFormatter extends Formatter { + } + export interface IpadicFeatures { + word_id: number; + word_type: string; + word_position: number; + surface_form: string; + pos: string; + pos_detail_1: string; + pos_detail_2: string; + pos_detail_3: string; + conjugated_type: string; + conjugated_form: string; + basic_form: string; + reading?: string; + pronunciation?: string; + } + + // viterbi/ViterbiBuilder.js interface ViterbiBuilder { - trie: any; - token_info_dictionary: any; - unknown_dictionary: any; + trie: doublearray.DoubleArray; + token_info_dictionary: TokenInfoDictionary; + unknown_dictionary: UnknownDictionary; build(sentence_str: string): ViterbiLattice; - - } - - interface ViterbiSearcher { - connection_costs: any; - search(lattice: ViterbiLattice): ViterbiNode[]; - forward(lattice: ViterbiLattice) } + // viterbi/ViterbiLattice.js interface ViterbiLattice { append(node: ViterbiNode): void; appendEos(): void; } + // viterbi/ViterbiNode.js interface ViterbiNode { name: string; cost: number; @@ -55,42 +124,37 @@ declare module kuromoji { type: string; } - interface IpadicFormatter { - formatEntry(word_id: number, position: number, type: string, features: string[]): IpadicFormat; - formatUnknownEntry(word_id: number, position: number, type: string, features: string[]): IpadicFormat; + // viterbi/ViterbiSearcher.js + interface ViterbiSearcher { + connection_costs: ConnectionCosts; + search(lattice: ViterbiLattice): ViterbiNode[]; + forward(lattice: ViterbiLattice): ViterbiLattice; + backward(lattice: ViterbiLattice): ViterbiNode[]; } - interface IpadicFormat { - word_id: number; - word_type: string; - word_position: number; - surface_form: number; - pos: string; - pos_detail_1: string; - pos_detail_2: string; - pos_detail_3: string; - conjugated_type: string; - conjugated_form: string; - basic_form: string; - reading?: string; - pronunciation?: string; + // Tokenizer.js + interface TokenizerStatic { + splitByPunctuation(input: string): string[]; + } + interface Tokenizer { + token_info_dictionary: TokenInfoDictionary; + unknown_dictionary: UnknownDictionary; + viterbi_builder: ViterbiBuilder; + viterbi_searcher: ViterbiSearcher; + formatter: Formatter; + tokenize(text: string): T[]; + getLattice(text: string): ViterbiLattice; } - interface DictionaryBuilder { - tid_entries: number[]; - unk_entries: number[]; - matrix_text: string; - char_text: string; - addTokenInfoDictionary(text: string): DictionaryBuilder; - costMatrix(matrix_text: string): DictionaryBuilder; - charDef(char_text: string): DictionaryBuilder; - unkDef(text: string): DictionaryBuilder; - build(): DynamicDictionaries; + // TokenizerBuilder.js + interface TokenizerBuilder { + build(callback: (err: Error, tokenizer: Tokenizer) => void): void; + } + interface TokenizerBuilderOption { + dicPath?: string; } - interface DynamicDictionaries { - } - - export function builder(option?: TokenizerBuilderOption): TokenizerBuilder; + // kuromoji.js + export function builder(option: TokenizerBuilderOption): TokenizerBuilder; export function dictionaryBuilder(): DictionaryBuilder; } \ No newline at end of file