gse
Go efficient text segmentation; support english, chinese, japanese and other.
Dictionary with double array trie (Double-Array Trie) to achieve, Sender algorithm is the shortest path based on word frequency plus dynamic programming, and DAG and HMM algorithm word segmentation.
Support common, search engine, full mode, precise mode and HMM mode multiple word segmentation modes, support user dictionary, POS tagging, run JSON RPC service.
Support HMM cut text use Viterbi algorithm.
Text Segmentation speed single thread 9.2MB/s，goroutines concurrent 26.8MB/s. HMM text segmentation single thread 3.2MB/s. (2core 4threads Macbook Pro).
Binding:
gse-bind, binding JavaScript and other, support more language.
Install / update
go get -u github.com/go-ego/gse
Build-tools
go get -u github.com/go-ego/re
re gse
To create a new gse application
$ re gse my-gse
re run
To run the application we just created, you can navigate to the application folder and execute:
$ cd my-gse && re run
Use
package main
import (
"fmt"
"github.com/go-ego/gse"
"github.com/go-ego/gse/hmm/pos"
)
var (
text = "Hello world, Helloworld. Winter is coming! 你好世界."
new = gse.New("zh,testdata/test_dict3.txt", "alpha")
seg gse.Segmenter
posSeg pos.Segmenter
)
func cut() {
hmm := new.Cut(text, true)
fmt.Println("cut use hmm: ", hmm)
hmm = new.CutSearch(text, true)
fmt.Println("cut search use hmm: ", hmm)
hmm = new.CutAll(text)
fmt.Println("cut all: ", hmm)
}
func main() {
cut()
segCut()
}
func posAndTrim(cut []string) {
cut = seg.Trim(cut)
fmt.Println("cut all: ", cut)
pos.WithGse(seg)
po := posSeg.Cut(text, true)
fmt.Println("pos: ", po)
po = posSeg.TrimWithPos(po, "zg")
fmt.Println("trim pos: ", po)
}
func cutPos() {
fmt.Println(seg.String(text, true))
fmt.Println(seg.Slice(text, true))
po := seg.Pos(text, true)
fmt.Println("pos: ", po)
po = seg.TrimPos(po)
fmt.Println("trim pos: ", po)
}
func segCut() {
// Loading the default dictionary
seg.LoadDict()
// Load the dictionary
// seg.LoadDict("your gopath"+"/src/github.com/go-ego/gse/data/dict/dictionary.txt")
// Text Segmentation
tb := []byte(text)
fmt.Println(seg.String(text, true))
segments := seg.Segment(tb)
// Handle word segmentation results
// Support for normal mode and search mode two participle,
// see the comments in the code ToString function.
// The search mode is mainly used to provide search engines
// with as many keywords as possible
fmt.Println(gse.ToString(segments, true))
}
Look at an custom dictionary example
package main
import (
"fmt"
"github.com/go-ego/gse"
)
func main() {
var seg gse.Segmenter
seg.LoadDict("zh,testdata/test_dict.txt,testdata/test_dict1.txt")
text1 := "你好世界, Hello world"
fmt.Println(seg.String(text1, true))
segments := seg.Segment([]byte(text1))
fmt.Println(gse.ToString(segments))
}
Authors
License
Gse is primarily distributed under the terms of both the MIT license and the Apache License (Version 2.0), thanks for sego and jieba(jiebago).
