Merge pull request #3377 from mzsm/kuromojijs

Japanese morphological analyzer "kuromoji.js"
2026-08-29 11:12:23 +08:00 · 2014-12-26 23:27:17 +09:00
parent 350a6abd9e fe562facd3
commit a8e45e394d
4 changed files with 324 additions and 0 deletions
@@ -0,0 +1,32 @@
+/// <reference path="doublearray.d.ts" />
+
+// https://github.com/takuyaa/doublearray/blob/master/README.md
+var words = [
+    { k: 'a', v: 1 },
+    { k: 'abc', v: 2 },
+    { k: '奈良', v: 3 },
+    { k: '奈良先端', v: 4 },
+    { k: '奈良先端科学技術大学院大学', v: 5 }
+];
+
+var trie = doublearray.builder().build(words);
+
+var trie = doublearray
+    .builder()
+    .append('a', 1)
+    .append('abc', 2)
+    .append('奈良', 3)
+    .append('奈良先端', 4)
+    .append('奈良先端科学技術大学院大学', 5)
+    .build();
+
+trie.contain('a');
+
+trie.lookup('abc');
+
+trie.commonPrefixSearch('奈良先端科学技術大学院大学');
+
+var base_buffer: Int32Array = trie.bc.getBaseBuffer();
+var check_buffer: Int32Array = trie.bc.getCheckBuffer();
+
+var loaded_trie = doublearray.load(base_buffer, check_buffer);
@@ -0,0 +1,99 @@
+// Type definitions for doublearray
+// Project: https://github.com/takuyaa/doublearray
+// Definitions by: MIZUSHIMA Junki <https://github.com/mzsm>
+// Definitions: https://github.com/borisyankov/DefinitelyTyped
+
+declare module doublearray {
+    interface KeyValue {
+        k: string;
+        v: number;
+    }
+
+    interface BaseAndCheck {
+        getBaseBuffer(): any;   // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array
+        getCheckBuffer(): any;  // Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array
+        loadBaseBuffer(base_buffer: Int8Array): BaseAndCheck;
+        loadBaseBuffer(base_buffer: Int16Array): BaseAndCheck;
+        loadBaseBuffer(base_buffer: Int32Array): BaseAndCheck;
+        loadBaseBuffer(base_buffer: Uint8Array): BaseAndCheck;
+        loadBaseBuffer(base_buffer: Uint16Array): BaseAndCheck;
+        loadBaseBuffer(base_buffer: Uint32Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Int8Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Int16Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Int32Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Uint8Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Uint16Array): BaseAndCheck;
+        loadCheckBuffer(check_buffer: Uint32Array): BaseAndCheck;
+        size(): number;
+        getBase(): number;
+        getCheck(): number;
+        setBase(index: number, base_value: number): void;
+        setCheck(index: number, check_value: number): void;
+        setFirstUnusedNode(index: number): void;
+        getFirstUnusedNode(): number;
+        shrink(): void;
+        calc(): {all: number; unused: number;  efficiency: number};
+        dump(): string;
+    }
+
+    interface DoubleArrayBuilder {
+        bc: BaseAndCheck;
+        keys: KeyValue[];
+        append(key: string, record: number): DoubleArrayBuilder;
+        build(keys?: KeyValue[], sorted?: boolean): DoubleArray;
+        getChildrenInfo(position: number, start: number, length: number): Int32Array;
+        setBC(parent_id: number, children_info: Int32Array, _base: number): void;
+        findAllocatableBase(children_info: Int32Array): number;
+        isUnusedNode(index: number): boolean;
+    }
+
+    interface DoubleArray {
+        bc: BaseAndCheck;
+        contain(key: string): boolean;
+        lookup(key: string): number;
+        commonPrefixSearch(key: string): KeyValue;
+        traverse(parent: number, code: number): number;
+        size(): number;
+        calc(): {all: number; unused: number;  efficiency: number};
+        dump(): string;
+    }
+
+    export function builder(initial_size?: number): DoubleArrayBuilder;
+    // TODO: Replace to union types in the future.
+    export function load(base_buffer: Int8Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Int8Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Int8Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Int8Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Int8Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Int8Array, check_buffer: Uint32Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Int16Array, check_buffer: Uint32Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Int32Array, check_buffer: Uint32Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Uint8Array, check_buffer: Uint32Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Uint16Array, check_buffer: Uint32Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Int8Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Int16Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Int32Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Uint8Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Uint16Array): DoubleArray;
+    export function load(base_buffer: Uint32Array, check_buffer: Uint32Array): DoubleArray;
+}
@@ -0,0 +1,33 @@
+/// <reference path="kuromoji.d.ts" />
+
+// From https://github.com/takuyaa/kuromoji.js/blob/master/README.md#usage
+kuromoji.builder({ dicPath: "/url/to/dictionary/dir/" }).build(function (err, tokenizer) {
+    var path = tokenizer.tokenize("すもももももももものうち");
+    var num_tmp: number;
+    var str_tmp: string;
+    path.forEach((token)=>{
+        num_tmp = token.word_id;
+        str_tmp = token.word_type;
+        num_tmp = token.word_position;
+        str_tmp = token.surface_form;
+        str_tmp = token.pos;
+        str_tmp = token.pos_detail_1;
+        str_tmp = token.pos_detail_2;
+        str_tmp = token.pos_detail_3;
+        str_tmp = token.conjugated_type;
+        str_tmp = token.conjugated_form;
+        str_tmp = token.basic_form;
+        str_tmp = token.reading;
+        str_tmp = token.pronunciation;
+    });
+});
+
+// From https://github.com/takuyaa/kuromoji.js/blob/master/test/resource/minimum-dic/minimum.csv
+var minimum_dict = [
+    "すもも,1285,1285,7546,名詞,一般,*,*,*,*,すもも,スモモ,スモモ",
+    "もも,1285,1285,7219,名詞,一般,*,*,*,*,もも,モモ,モモ"
+].join('\n');
+
+var builder = kuromoji.dictionaryBuilder();
+builder = builder.addTokenInfoDictionary(minimum_dict);
+var dict = builder.build();
@@ -0,0 +1,160 @@
+// Type definitions for kuromoji.js
+// Project: https://github.com/takuyaa/kuromoji.js
+// Definitions by: MIZUSHIMA Junki <https://github.com/mzsm>
+// Definitions: https://github.com/borisyankov/DefinitelyTyped
+
+/// <reference path="../doublearray/doublearray.d.ts" />
+
+declare module kuromoji {
+
+    // dict/ConnectionCosts.js
+    interface ConnectionCosts {
+        buffer: Int16Array;
+        put(forward_id: number, backward_id: number, cost: number): void;
+        get(forward_id: number, backward_id: number): number;
+        loadConnectionCosts(connection_costs_buffer: Int16Array): void;
+    }
+
+    // dict/DynamicDictionaries.js
+    interface DynamicDictionaries {
+        trie: doublearray.DoubleArray;
+        token_info_dictionary: TokenInfoDictionary;
+        connection_costs: ConnectionCosts;
+        unknown_dictionary: UnknownDictionary;
+        loadTrie(base_buffer: Int32Array, check_buffer: Int32Array): DynamicDictionaries;
+    }
+
+    // dict/TokenInfoDictionary.js
+    interface TokenInfoDictionary {
+        buildDictionary(entries: any[][]): {[word_id: number]: string};
+        put(left_id: number, right_id: number, word_cost: number, surface_form: string, feature: string): number;
+        addMapping(source: number, target: number): void;
+        targetMapToBuffer(): Uint8Array;
+        loadDictionary(array_buffer: Uint8Array): TokenInfoDictionary;
+        loadPosVector(array_buffer: Uint8Array): TokenInfoDictionary;
+        loadTargetMap(array_buffer: Uint8Array): TokenInfoDictionary;
+        getFeatures(token_info_id_str: string): string;
+    }
+
+    // dict/UnknownDictionary.js
+    interface UnknownDictionary extends TokenInfoDictionary {
+    }
+
+    // util/ByteBuffer.js
+    interface ByteBuffer {
+        buffer: Uint8Array;
+        position: number;
+        size(): number;
+        reallocate(): void;
+        shrink(): Uint8Array;
+        put(b: number): void;
+        get(index: number): number;
+        putShort(num: number): void;
+        getShort(index: number): number;
+        putInt(num: number): void;
+        getInt(index: number): number;
+        readInt(): number;
+        putString(str: string): void;
+        getString(index: number): string;
+    }
+
+    // util/DictionaryBuilder.js
+    interface DictionaryBuilder {
+        tid_entries: string[];
+        unk_entries: string[];
+        addTokenInfoDictionary(text: string): DictionaryBuilder;
+        costMatrix(matrix_text: string): DictionaryBuilder;
+        charDef(char_text: string): DictionaryBuilder;
+        unkDef(text: string): DictionaryBuilder;
+        build(): DynamicDictionaries;
+        buildTokenInfoDictionary(): {trie: doublearray.DoubleArray; token_info_dictionary: TokenInfoDictionary};
+        buildUnknownDictionary(): UnknownDictionary;
+        buildConnectionCosts(): ConnectionCosts;
+        buildDoubleArray(): doublearray.DoubleArray;
+    }
+
+    // util/IpadicFormatter.js
+    interface Formatter<T> {
+        formatEntry(word_id: number, position: number, type: string, features: string[]): T;
+        formatUnknownEntry(word_id: number, position: number, type: string, features: string[]): T;
+    }
+    interface IpadicFormatter extends Formatter<IpadicFeatures> {
+    }
+    export interface IpadicFeatures {
+        word_id: number;
+        word_type: string;
+        word_position: number;
+        surface_form: string;
+        pos: string;
+        pos_detail_1: string;
+        pos_detail_2: string;
+        pos_detail_3: string;
+        conjugated_type: string;
+        conjugated_form: string;
+        basic_form: string;
+        reading?: string;
+        pronunciation?: string;
+    }
+
+    // viterbi/ViterbiBuilder.js
+    interface ViterbiBuilder {
+        trie: doublearray.DoubleArray;
+        token_info_dictionary: TokenInfoDictionary;
+        unknown_dictionary: UnknownDictionary;
+        build(sentence_str: string): ViterbiLattice;
+    }
+
+    // viterbi/ViterbiLattice.js
+    interface ViterbiLattice {
+        append(node: ViterbiNode): void;
+        appendEos(): void;
+    }
+
+    // viterbi/ViterbiNode.js
+    interface ViterbiNode {
+        name: string;
+        cost: number;
+        start_pos: number;
+        length: number;
+        left_id: number;
+        right_id: number;
+        prev: ViterbiNode;
+        surface_form: string;
+        shortest_cost: number;
+        type: string;
+    }
+
+    // viterbi/ViterbiSearcher.js
+    interface ViterbiSearcher {
+        connection_costs: ConnectionCosts;
+        search(lattice: ViterbiLattice): ViterbiNode[];
+        forward(lattice: ViterbiLattice): ViterbiLattice;
+        backward(lattice: ViterbiLattice): ViterbiNode[];
+    }
+
+    // Tokenizer.js
+    interface TokenizerStatic {
+        splitByPunctuation(input: string): string[];
+    }
+    interface Tokenizer<T> {
+        token_info_dictionary: TokenInfoDictionary;
+        unknown_dictionary: UnknownDictionary;
+        viterbi_builder: ViterbiBuilder;
+        viterbi_searcher: ViterbiSearcher;
+        formatter: Formatter<T>;
+        tokenize(text: string): T[];
+        getLattice(text: string): ViterbiLattice;
+    }
+
+    // TokenizerBuilder.js
+    interface TokenizerBuilder<T> {
+        build(callback: (err: Error, tokenizer: Tokenizer<T>) => void): void;
+    }
+    interface TokenizerBuilderOption {
+        dicPath?: string;
+    }
+
+    // kuromoji.js
+    export function builder(option: TokenizerBuilderOption): TokenizerBuilder<IpadicFeatures>;
+    export function dictionaryBuilder(): DictionaryBuilder;
+}