kiwi

package module
v0.0.0-...-ba165a3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 12, 2026 License: LGPL-2.1 Imports: 6 Imported by: 0

README

kiwigo

CI Go Reference

https://github.com/bab2min/Kiwi for go

Dependencies

  • Install Kiwi

check out how to install Kiwi here

Example script (install using the helper script)
make install-kiwi
Example script (install from source)
git clone [email protected]:bab2min/Kiwi.git
cd Kiwi
git submodule sync
git submodule update --init --recursive
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ../
make
make install
ldconfig

Documentation

Overview

Package kiwi is a Go binding for Kiwi (https://github.com/bab2min/Kiwi) project.

Example 
package main

import (
	"fmt"

	kiwi "github.com/codingpot/kiwigo"
)

func main() {
	kb, err := kiwi.NewBuilder("./base", kiwi.WithNumThread(1), kiwi.WithBuildOption(kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH))
	if err != nil {
		panic(err)
	}
	kb.AddWord("코딩냄비", "NNP", 0)

	k := kb.Build()
	defer k.Close() // don't forget to Close()!

	results, _ := k.Analyze("안녕하세요 코딩냄비입니다. 부글부글.")

	// Print tokens without the score to avoid floating-point output issues
	if len(results) > 0 {
		fmt.Printf("Tokens: %v\n", results[0].Tokens)
	}
}

Output:
Tokens: [{0 NNG 안녕} {2 XSA 하} {3 EF 세요} {6 NNP 코딩냄비} {10 VCP 이} {10 EF ᆸ니다} {13 SF .} {15 MAG 부글부글} {19 SF .}]

Index

Examples

Constants

View Source 
const (
	// Default values derived from Kiwi C-API defaults (include/kiwi/capi.h).
	// For detailed information on these parameters, refer to:
	// https://github.com/bab2min/Kiwi/blob/main/include/kiwi/capi.h
	DefaultDialectCost   float32 = 3.0 // Default penalty for dialect words (dialect_cost)
	DefaultTypoThreshold float32 = 2.5 // Default cost threshold for typo correction (typo_threshold)
	DefaultNumThread     int     = 0   // Default number of threads (0 means auto-detect based on CPU cores)
	DefaultTopN          int     = 1   // Default number of results to return from Analyze
)
View Source 
const (
	POS_USER_0 = POS_USER0
	POS_USER_1 = POS_USER1
	POS_USER_2 = POS_USER2
	POS_USER_3 = POS_USER3
	POS_USER_4 = POS_USER4
)

Deprecated: these aliases are kept for backwards compatibility.

Variables

This section is empty.

Functions

func KiwiClearError 

func KiwiClearError()

KiwiClearError clear error.

func KiwiError 

func KiwiError() string

KiwiError returns the Error messages.

func KiwiReaderImpl 

func KiwiReaderImpl(lineNumber C.int, buffer *C.char, userData unsafe.Pointer) C.int

func KiwiVersion 

func KiwiVersion() string

KiwiVersion returns the version of the kiwi library.

Types

type AnalyzeOptionFunc 

type AnalyzeOptionFunc func(*AnalyzeOptions)

AnalyzeOptionFunc represents a configuration function for Analyze.

func WithAllowedDialects 

func WithAllowedDialects(dialects Dialect) AnalyzeOptionFunc

WithAllowedDialects sets the allowed dialects for Analyze.

func WithBlocklist 

func WithBlocklist(blocklist *Morphset) AnalyzeOptionFunc

WithBlocklist sets the blocklist for Analyze.

func WithDialectCost 

func WithDialectCost(cost float32) AnalyzeOptionFunc

WithDialectCost sets the dialect cost for Analyze.

func WithMatchOption 

func WithMatchOption(options MatchOption) AnalyzeOptionFunc

WithMatchOption sets the MatchOption for Analyze.

func WithOpenEnding 

func WithOpenEnding(openEnding bool) AnalyzeOptionFunc

WithOpenEnding sets whether to keep the sentence open after the last morpheme.

func WithTopN 

func WithTopN(n int) AnalyzeOptionFunc

WithTopN sets the maximum number of results to return from Analyze.

func WithTypoThreshold 

func WithTypoThreshold(threshold float32) AnalyzeOptionFunc

WithTypoThreshold sets the typo threshold for Analyze.

type AnalyzeOptions 

type AnalyzeOptions struct {
	MatchOptions    MatchOption
	Blocklist       *Morphset
	OpenEnding      bool
	AllowedDialects Dialect
	DialectCost     float32
	TypoThreshold   float32
	TopN            int
}

AnalyzeOptions provides configuration for the Analyze function.

func DefaultAnalyzeOptions 

func DefaultAnalyzeOptions() AnalyzeOptions

DefaultAnalyzeOptions returns the default AnalyzeOptions recommended by Kiwi.

type BuildOption 

type BuildOption int

BuildOption is a bitwise OR of the KiwiBuildOption values. 

const (
	KIWI_BUILD_INTEGRATE_ALLOMORPH BuildOption = C.KIWI_BUILD_INTEGRATE_ALLOMORPH
	KIWI_BUILD_LOAD_DEFAULT_DICT   BuildOption = C.KIWI_BUILD_LOAD_DEFAULT_DICT
	KIWI_BUILD_LOAD_TYPO_DICT      BuildOption = C.KIWI_BUILD_LOAD_TYPO_DICT
	KIWI_BUILD_LOAD_MULTI_DICT     BuildOption = C.KIWI_BUILD_LOAD_MULTI_DICT
	KIWI_BUILD_DEFAULT             BuildOption = C.KIWI_BUILD_DEFAULT

	// Model type is a single-select field, not a bitmask.
	// Select exactly one of the following mutually exclusive model types:
	//
	//   - MODEL_TYPE_DEFAULT: default model (0x0000)
	//   - MODEL_TYPE_LARGEST: largest available model
	//   - MODEL_TYPE_KNLM: KNLM model (deprecated)
	//   - MODEL_TYPE_SBG: SBG model (deprecated)
	//   - MODEL_TYPE_CONG: CoNg model
	//   - MODEL_TYPE_CONG_GLOBAL: CoNg global model
	//
	// Do NOT combine these with bitwise OR.
	KIWI_BUILD_MODEL_TYPE_DEFAULT     BuildOption = C.KIWI_BUILD_MODEL_TYPE_DEFAULT
	KIWI_BUILD_MODEL_TYPE_LARGEST     BuildOption = C.KIWI_BUILD_MODEL_TYPE_LARGEST
	KIWI_BUILD_MODEL_TYPE_KNLM        BuildOption = C.KIWI_BUILD_MODEL_TYPE_KNLM
	KIWI_BUILD_MODEL_TYPE_SBG         BuildOption = C.KIWI_BUILD_MODEL_TYPE_SBG
	KIWI_BUILD_MODEL_TYPE_CONG        BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG
	KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL
)

type Config 

type Config struct {
	IntegrateAllomorph             bool
	CutOffThreshold                float32
	OovRuleScale                   float32
	OovRuleBias                    float32
	OovChrBias                     float32
	OovGlobalWeight                float32
	OovLocalWeight                 float32
	OovGlobalMinFreq               float32
	SpacePenalty                   float32
	TypoCostWeight                 float32
	MaxUnkFormSize                 uint32
	MaxUnkFormSizeFollowedByJClass uint32
	SpaceTolerance                 uint32
}

Config represents the configuration for Kiwi analysis.

type Dialect 

type Dialect int

Dialect represents a dialect in the Kiwi API. 

const (
	DialectStandard    Dialect = 0 // KIWI_DIALECT_STANDARD
	DialectGyeonggi    Dialect = 1 << 0
	DialectChungcheong Dialect = 1 << 1
	DialectGangwon     Dialect = 1 << 2
	DialectGyeongsang  Dialect = 1 << 3
	DialectJeolla      Dialect = 1 << 4
	DialectJeju        Dialect = 1 << 5
	DialectHwanghae    Dialect = 1 << 6
	DialectHamgyeong   Dialect = 1 << 7
	DialectPyeongan    Dialect = 1 << 8
	DialectArchaic     Dialect = 1 << 9
	DialectAll         Dialect = (1<<9)*2 - 1
)

type Kiwi 

type Kiwi struct {
	// contains filtered or unexported fields
}

Kiwi is a wrapper for the kiwi C library.

Thread Safety: Concurrent calls to Analyze are safe. However, SetGlobalConfig must not be called concurrently with Analyze or other methods that read the configuration.

func New 

func New(modelPath string, opts ...Option) (*Kiwi, error)

New returns a new Kiwi instance. Don't forget to call Close after this.

func (*Kiwi) Analyze 

func (k *Kiwi) Analyze(text string, opts ...AnalyzeOptionFunc) ([]TokenResult, error)

Analyze returns the result of the analysis.

func (*Kiwi) Close 

func (k *Kiwi) Close() int

Close frees the resource allocated for Kiwi and returns the exit status. This must be called after New. Returns 0 if successful. Safe to call multiple times.

func (*Kiwi) GetGlobalConfig 

func (k *Kiwi) GetGlobalConfig() Config

GetGlobalConfig returns the global configuration of the Kiwi instance.

func (*Kiwi) GetOption 

func (k *Kiwi) GetOption(option OptionType) int

GetOption returns the int value of the specified option.

func (*Kiwi) GetOptionF 

func (k *Kiwi) GetOptionF(option OptionType) float32

GetOptionF returns the float value of the specified option. Note: As of Kiwi v0.23.2, there are no float options available. This function is provided for future compatibility.

func (*Kiwi) NewMorphset 

func (k *Kiwi) NewMorphset() (*Morphset, error)

NewMorphset creates a new morpheme set. The Morphset must be closed after use, before the parent Kiwi instance is closed.

func (*Kiwi) SetGlobalConfig 

func (k *Kiwi) SetGlobalConfig(config Config)

SetGlobalConfig sets the global configuration of the Kiwi instance.

func (*Kiwi) SetOption 

func (k *Kiwi) SetOption(option OptionType, value int)

SetOption sets the int value of the specified option.

func (*Kiwi) SetOptionF 

func (k *Kiwi) SetOptionF(option OptionType, value float32)

SetOptionF sets the float value of the specified option. Note: As of Kiwi v0.23.2, there are no float options available. This function is provided for future compatibility.

func (*Kiwi) SplitSentence 

func (k *Kiwi) SplitSentence(text string, options MatchOption) ([]SplitResult, error)

SplitSentence returns the line of sentences.

type KiwiBuilder 

type KiwiBuilder struct {
	// contains filtered or unexported fields
}

KiwiBuilder is a wrapper for the kiwi C library.

func NewBuilder 

func NewBuilder(modelPath string, opts ...Option) (*KiwiBuilder, error)

NewBuilder returns a new KiwiBuilder instance. Don't forget to call Close after this.

func (*KiwiBuilder) AddWord 

func (kb *KiwiBuilder) AddWord(word string, pos POSType, score float32) int

AddWord set custom word with word, pos, score.

func (*KiwiBuilder) Build 

func (kb *KiwiBuilder) Build() *Kiwi

Build creates kiwi instance with user word etc.

func (*KiwiBuilder) Close 

func (kb *KiwiBuilder) Close() int

Close frees the resource allocated for KiwiBuilder and returns the exit status. This must be called after New but not need to called after Build. Returns 0 if successful. Safe to call multiple times.

func (*KiwiBuilder) ExtractWords 

func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWordLen int, minScore float32, posThreshold float32) ([]WordInfo, error)

ExtractWords returns the result of extract word.

func (*KiwiBuilder) LoadDict 

func (kb *KiwiBuilder) LoadDict(dictPath string) int

LoadDict loads user dict with dict file path.

type MatchOption 

type MatchOption int

MatchOption is a bitwise OR of the KiwiMatchOption values. 

const (
	KIWI_MATCH_URL     MatchOption = C.KIWI_MATCH_URL
	KIWI_MATCH_EMAIL   MatchOption = C.KIWI_MATCH_EMAIL
	KIWI_MATCH_HASHTAG MatchOption = C.KIWI_MATCH_HASHTAG
	KIWI_MATCH_MENTION MatchOption = C.KIWI_MATCH_MENTION
	KIWI_MATCH_SERIAL  MatchOption = C.KIWI_MATCH_SERIAL
	KIWI_MATCH_EMOJI   MatchOption = C.KIWI_MATCH_EMOJI

	// OOV detection mode is a 2-bit field (bits 8-9), not independent flags.
	// Select exactly one of the following mutually exclusive modes:
	//
	//   - OOV_RULE_ONLY: rule-based scoring (default, value 0 << 8)
	//   - OOV_CHR_MODEL: character model-based scoring
	//   - OOV_CHR_FREQ_MODEL: character + frequency model scoring
	//   - OOV_CHR_FREQ_BRANCH_MODEL: character + frequency + branch model scoring
	//
	// Do NOT combine these with bitwise OR. KIWI_MATCH_OOV_MASK is for
	// internal use and should not be selected directly.
	KIWI_MATCH_OOV_RULE_ONLY             MatchOption = C.KIWI_MATCH_OOV_RULE_ONLY
	KIWI_MATCH_OOV_CHR_MODEL             MatchOption = C.KIWI_MATCH_OOV_CHR_MODEL
	KIWI_MATCH_OOV_CHR_FREQ_MODEL        MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_MODEL
	KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL
	KIWI_MATCH_OOV_MASK                  MatchOption = C.KIWI_MATCH_OOV_MASK

	KIWI_MATCH_NORMALIZE_CODA   MatchOption = C.KIWI_MATCH_NORMALIZE_CODA
	KIWI_MATCH_JOIN_NOUN_PREFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_PREFIX
	KIWI_MATCH_JOIN_NOUN_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_SUFFIX
	KIWI_MATCH_JOIN_VERB_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_VERB_SUFFIX
	KIWI_MATCH_JOIN_ADJ_SUFFIX  MatchOption = C.KIWI_MATCH_JOIN_ADJ_SUFFIX
	KIWI_MATCH_JOIN_ADV_SUFFIX  MatchOption = C.KIWI_MATCH_JOIN_ADV_SUFFIX
	KIWI_MATCH_JOIN_V_SUFFIX    MatchOption = C.KIWI_MATCH_JOIN_V_SUFFIX
	KIWI_MATCH_JOIN_AFFIX       MatchOption = C.KIWI_MATCH_JOIN_AFFIX
	KIWI_MATCH_SPLIT_COMPLEX    MatchOption = C.KIWI_MATCH_SPLIT_COMPLEX
	KIWI_MATCH_Z_CODA           MatchOption = C.KIWI_MATCH_Z_CODA
	KIWI_MATCH_COMPATIBLE_JAMO  MatchOption = C.KIWI_MATCH_COMPATIBLE_JAMO
	KIWI_MATCH_SPLIT_SAISIOT    MatchOption = C.KIWI_MATCH_SPLIT_SAISIOT
	KIWI_MATCH_MERGE_SAISIOT    MatchOption = C.KIWI_MATCH_MERGE_SAISIOT
	KIWI_MATCH_JOIN_PARTICLE_YO MatchOption = C.KIWI_MATCH_JOIN_PARTICLE_YO
	KIWI_MATCH_USE_OLD_SPLITTER MatchOption = C.KIWI_MATCH_USE_OLD_SPLITTER

	KIWI_MATCH_ALL                  MatchOption = C.KIWI_MATCH_ALL
	KIWI_MATCH_ALL_WITH_NORMALIZING MatchOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING
)

type Morphset 

type Morphset struct {
	// contains filtered or unexported fields
}

Morphset represents a set of morphemes that can be used as a blocklist.

func (*Morphset) Add 

func (ms *Morphset) Add(form string, tag string) (int, error)

Add adds a morpheme to the set. tag is a POS tag such as "NNG". If tag is empty, all morphemes matching form are added. Returns the number of morphemes added, or an error.

func (*Morphset) Close 

func (ms *Morphset) Close()

Close frees the resources allocated for the Morphset. Must be called before the parent Kiwi instance is closed.

type Option 

type Option func(*kiwiOptions)

Option represents a configuration function for Kiwi initialization.

func WithBuildOption 

func WithBuildOption(options BuildOption) Option

WithBuildOption sets the BuildOption for initialization.

func WithDialect 

func WithDialect(dialects Dialect) Option

WithDialect sets the allowed dialects for initialization.

func WithNumThread 

func WithNumThread(threads int) Option

WithNumThread sets the number of threads for initialization. A value of 0 tells Kiwi to automatically use all available CPU cores.

type OptionType 

type OptionType int

OptionType represents the type of option for kiwi_set_option/kiwi_get_option. 

const (
	KIWI_NUM_THREADS OptionType = C.KIWI_NUM_THREADS
)

type POSType 

type POSType string
const (
	POS_UNKNOWN   POSType = "UN"
	POS_NNG       POSType = "NNG"
	POS_NNP       POSType = "NNP"
	POS_NNB       POSType = "NNB"
	POS_VV        POSType = "VV"
	POS_VA        POSType = "VA"
	POS_MAG       POSType = "MAG"
	POS_NR        POSType = "NR"
	POS_NP        POSType = "NP"
	POS_VX        POSType = "VX"
	POS_MM        POSType = "MM"
	POS_MAJ       POSType = "MAJ"
	POS_IC        POSType = "IC"
	POS_XPN       POSType = "XPN"
	POS_XSN       POSType = "XSN"
	POS_XSV       POSType = "XSV"
	POS_XSA       POSType = "XSA"
	POS_XSM       POSType = "XSM"
	POS_XR        POSType = "XR"
	POS_VCP       POSType = "VCP"
	POS_VCN       POSType = "VCN"
	POS_SF        POSType = "SF"
	POS_SP        POSType = "SP"
	POS_SS        POSType = "SS"
	POS_SSO       POSType = "SSO"
	POS_SSC       POSType = "SSC"
	POS_SE        POSType = "SE"
	POS_SO        POSType = "SO"
	POS_SW        POSType = "SW"
	POS_SB        POSType = "SB"
	POS_SL        POSType = "SL"
	POS_SH        POSType = "SH"
	POS_SN        POSType = "SN"
	POS_W_URL     POSType = "W_URL"
	POS_W_EMAIL   POSType = "W_EMAIL"
	POS_W_MENTION POSType = "W_MENTION"
	POS_W_HASHTAG POSType = "W_HASHTAG"
	POS_W_SERIAL  POSType = "W_SERIAL"
	POS_W_EMOJI   POSType = "W_EMOJI"
	POS_JKS       POSType = "JKS"
	POS_JKC       POSType = "JKC"
	POS_JKG       POSType = "JKG"
	POS_JKO       POSType = "JKO"
	POS_JKB       POSType = "JKB"
	POS_JKV       POSType = "JKV"
	POS_JKQ       POSType = "JKQ"
	POS_JX        POSType = "JX"
	POS_JC        POSType = "JC"
	POS_EP        POSType = "EP"
	POS_EF        POSType = "EF"
	POS_EC        POSType = "EC"
	POS_ETN       POSType = "ETN"
	POS_ETM       POSType = "ETM"
	POS_Z_CODA    POSType = "Z_CODA"
	POS_Z_SIOT    POSType = "Z_SIOT"
	POS_USER0     POSType = "USER0"
	POS_USER1     POSType = "USER1"
	POS_USER2     POSType = "USER2"
	POS_USER3     POSType = "USER3"
	POS_USER4     POSType = "USER4"
	POS_P         POSType = "P"
	POS_PV        POSType = "P"
	POS_VV_I      POSType = "VV-I"
	POS_VA_I      POSType = "VA-I"
	POS_VX_I      POSType = "VX-I"
	POS_XSA_I     POSType = "XSA-I"
	POS_VV_R      POSType = "VV-R"
	POS_VA_R      POSType = "VA-R"
	POS_VX_R      POSType = "VX-R"
	POS_XSA_R     POSType = "XSA-R"

	// Accepted by toPOSTag but never produced by Kiwi itself.
	POS_NF    POSType = "NF"
	POS_NV    POSType = "NV"
	POS_NA    POSType = "NA"
	POS_UNK   POSType = "UNK"
	POS_V     POSType = "V"
	POS_A     POSType = "A"
	POS_CARET POSType = "^"
)

func ParsePOSType 

func ParsePOSType(t string) (POSType, error)

ParsePOSType return POS Tag for result

type SplitResult 

type SplitResult struct {
	Text  string
	Begin int
	End   int
}

SplitResult returns the Sentences.

type TokenInfo 

type TokenInfo struct {
	// Position is the index of this token appears in the original text.
	Position int

	// Tag represents a type of this token (e.g. VV, NNG, ...).
	Tag POSType

	// Form is the actual string of this token.
	Form string
}

TokenInfo returns the token info for the given token(Str).

type TokenResult 

type TokenResult struct {
	Tokens []TokenInfo
	Score  float32
}

TokenResult is a result for Analyze.

type WordInfo 

type WordInfo struct {
	Form     string
	Freq     int
	POSScore float32
	Score    float32
}

WordInfo returns the token info for the given token(Str).

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.