diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a09c56d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.idea diff --git a/CHANGELOG.phoneatatool.md b/CHANGELOG.phoneatatool.md new file mode 100644 index 0000000..16b70bb --- /dev/null +++ b/CHANGELOG.phoneatatool.md @@ -0,0 +1,24 @@ +# CHANGELOG.phoneatatool + +## [0.2.0] - 2023-05-21 + +### 新增 + +- 实现了号码查询功能。 + +### 改动 + +- 说明文档更清晰了。 + +### 修复 + +## [0.1.0] - 2023-05-21 + +### 新增 + +- 实现了打包功能。 +- 实习了解包功能。 + +### 改动 + +### 修复 diff --git a/README.phonedatatool.md b/README.phonedatatool.md new file mode 100644 index 0000000..ff2a161 --- /dev/null +++ b/README.phonedatatool.md @@ -0,0 +1,120 @@ +# README.phonedatatool.md + +工具 phonedatatool 用于对给定 phone.dat 的解包和打包。 + +## 1. 使用方法 + +- 第一步：将原始二进制文件解包成文本。 +- 第二步：根据需要手动修改文本。 +- 第三步：将修改后的文本打包成二进制文件。 +- 第四步：用修改后的二进制文件查询号码。 + +## 2. 解包 + +```shell +D:\seedjyh\phonedata>phonedatatool.exe -unpack -i phone.dat -o tmp +Unpack completed. +``` + +可以将二进制文件 phone.dat 解包到一个名为 abc 的目录。里面有三个文本文件。 + +## 3. 打包 + +```shell +D:\seedjyh\phonedata>phonedatatool.exe -pack -i tmp -o phone.2.dat +Pack completed. +``` + +可以将目录 abc 里的文本文件打包成二进制文件 phone.2.dat + +## 4. 查询号码 + +```shell +D:\seedjyh\phonedata>phonedatatool.exe -query -i phone.2.dat -number 13336061916 +PhoneNum: 13336061916 +AreaZone: 0571 +CardType: 中国电信 +City: 杭州 +ZipCode: 310000 +Province: 浙江 +Query completed. +``` + +## 5. 解包后文件说明 + +解包后的目录下会产生 3 个文本文件，功能分别是： + +| 文件名 | 原始二进制文件 | +| ----------- | --------------------------------------------- | +| version.txt | 版本号 | +| record.txt | 记录区（省、市、邮编、区号） | +| index.txt | 索引区（号码前 7 位、记录区偏移量、号码类型） | + +### 5.1. version.txt + +里面应该是 4 个字符。比如 "2307"。 + +### 5.2. record.txt + +有多行，每行包括一条记录，例如`1|安徽|巢湖|238000|0551`。 + +由竖线分隔成 5 段，含义分别是： + +| 例子 | 含义 | +| ------ | --------- | +| 1 | 记录区 ID | +| 安徽 | 省名 | +| 巢湖 | 城市名 | +| 238000 | 邮政编码 | +| 0551 | 区号 | + +其中「记录区 ID」必须是整数，不一定要连续，但每行的「记录区 ID」必须不同。 + +### 5.3. index.txt + +有多行，每行包括一条索引，例如`1300000|251|2` + +由竖线分隔成 3 段，含义分别是： + +| 例子 | 含义 | +| ------- | ----------------------------------- | +| 1300000 | 手机号码前 7 位 | +| 251 | 记录区 ID（含义见 record.txt 章节） | +| 2 | 卡片类型码 | + +### 5.4. 文本文件修改方式 + +#### 5.4.1. 新增一个号码段 + +如果一个号码段（前七位）在索引文件 index.txt 里没有，则可以直接在 index.txt 末尾加入一条记录。 + +如果这个号段前缀归属于 record.txt 里已有的某一地区，则「记录区 ID」直接填该地区的 ID 即可。 + +否则，需要先在 record.txt 里添加一条记录（注意新记录的 ID 必须和已有的所有 ID 均不相同），然后再往 index.txt 里添加记录。 + +#### 5.4.2. 修改一个号码段 + +直接修改该号码段在 index.txt 里的信息即可。例如，将「记录区 ID」修改成另一个数。必要时也要先在 record.txt 里新增记录。 + +#### 5.4.3. 删除一个号码段 + +直接删除 index.txt 里的信息即可。 + +#### 5.4.4. 备注 + +所有文本文件都必须以换行符结尾。 + +## 6. 备注 + +### 6.1. 卡片类型码 + +卡片类型码的含义由 phonedata 原项目规定。目前（2023-05-24）类型码的含义为： + +| 卡片类型码 | 卡片类型 | +| ---------- | -------------- | +| 1 | 中国移动 | +| 2 | 中国联通 | +| 3 | 中国电信 | +| 4 | 电信虚拟运营商 | +| 5 | 联通虚拟运营商 | +| 6 | 移动虚拟运营商 | diff --git a/cmd/phonedatatool/main.go b/cmd/phonedatatool/main.go new file mode 100644 index 0000000..4006efe --- /dev/null +++ b/cmd/phonedatatool/main.go @@ -0,0 +1,197 @@ +package main + +import ( + "flag" + "fmt" + "github.com/xluohome/phonedata/phonedatatool/pack" + "github.com/xluohome/phonedata/phonedatatool/util" + "os" + "path" +) + +// 这里编译出来的可执行程序具备打包、查询、解包三个功能。 +// ./phonedatatool -unpack -i phone.dat -o tmp +// ./phonedatatool -pack -i tmp -o phone.dat +// ./phonedatatool -query -i phone.dat -number 13000001234 + +const ( + Name = "phonedatatool" + Version = "0.2.0" + Author = "seedjyh@gmail.com" + FullName = Name + "_" + Version + "(" + Author + ")" +) + +const ( + VersionFileName = "version.txt" + RecordFileName = "record.txt" + IndexFileName = "index.txt" +) + +func main() { + showVersionFlag := flag.Bool("v", false, "Just show version string") + showHelpFlag := flag.Bool("help", false, "Just print help.") + unpackFlag := flag.Bool("unpack", false, "Unpack phone data to plain text") + packFlag := flag.Bool("pack", false, "Pack plain text to phone data") + queryFlag := flag.Bool("query", false, "Query number from phone data") + source := flag.String("i", "", "Source of operation") + destination := flag.String("o", "", "Destination of operation") + number := flag.String("number", "", "Number to query") + flag.Parse() + if *showVersionFlag { + fmt.Println("Version:", FullName) + return + } + if *showHelpFlag { + showHelp() + return + } + if *unpackFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if destination == nil { + fmt.Println("ERROR! No destination") + return + } + if err := Unpack(*source, *destination); err != nil { + fmt.Println("ERROR! Unpack failed.", err) + return + } else { + fmt.Println("Unpack completed.") + return + } + } + if *packFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if destination == nil { + fmt.Println("ERROR! No destination") + return + } + if err := Pack(*source, *destination); err != nil { + fmt.Println("ERROR! Pack failed.", err) + return + } else { + fmt.Println("Pack completed.") + return + } + } + if *queryFlag { + if source == nil { + fmt.Println("ERROR! No source") + return + } + if number == nil { + fmt.Println("ERROR! No number to query") + return + } + if err := QueryNumber(*source, *number); err != nil { + fmt.Println("ERROR! Query failed.", err) + return + } else { + fmt.Println("Query completed.") + return + } + } + fmt.Println("Did nothing.") + showHelp() + return +} + +func showHelp() { + fmt.Println("<< HELP >>") + fmt.Println("./phonedatatool -unpack -i phone.dat -o tmp") + fmt.Println("./phonedatatool -pack -i tmp -o phone.dat") + fmt.Println("./phonedatatool -query -i phone.dat -n 13000001234") +} + +func Pack(plainDirectoryPath string, phoneDataFilePath string) error { + if err := util.AssureFileNotExist(phoneDataFilePath); err != nil { + return err + } + var versionPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, VersionFileName)); err != nil { + return err + } else { + versionPlainTextBuf = buf + } + + var recordPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, RecordFileName)); err != nil { + return err + } else { + recordPlainTextBuf = buf + } + + var indexPlainTextBuf []byte + if buf, err := os.ReadFile(path.Join(plainDirectoryPath, IndexFileName)); err != nil { + return err + } else { + indexPlainTextBuf = buf + } + + if buf, err := pack.NewPacker().Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf); err != nil { + return err + } else { + return os.WriteFile(phoneDataFilePath, buf, 0) + } +} + +func Unpack(phoneDataFilePath string, plainDirectoryPath string) error { + if err := os.MkdirAll(plainDirectoryPath, 0); err != nil { + return fmt.Errorf("target directory %v not exist and can't be created: %v", plainDirectoryPath, err) + } + + versionFilePath := path.Join(plainDirectoryPath, VersionFileName) + recordFilePath := path.Join(plainDirectoryPath, RecordFileName) + indexFilePath := path.Join(plainDirectoryPath, IndexFileName) + + if err := util.AssureAllFileNotExist(versionFilePath, recordFilePath, indexFilePath); err != nil { + return err + } + + var rawBuf []byte + if b, err := os.ReadFile(phoneDataFilePath); err != nil { + return err + } else { + rawBuf = b + } + + if versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf, err := pack.NewUnpacker().Unpack(rawBuf); err != nil { + return err + } else { + if err := os.WriteFile(versionFilePath, versionPlainTextBuf, 0); err != nil { + return err + } + if err := os.WriteFile(recordFilePath, recordPlainTextBuf, 0); err != nil { + return err + } + if err := os.WriteFile(indexFilePath, indexPlainTextBuf, 0); err != nil { + return err + } + return nil + } +} + +func QueryNumber(phoneDataFilePath string, number string) error { + var rawBuf []byte + if b, err := os.ReadFile(phoneDataFilePath); err != nil { + return err + } else { + rawBuf = b + } + if info, err := pack.NewQuerier().Query(rawBuf, number); err != nil { + return err + } else { + fmt.Println("PhoneNum: ", info.PhoneNumber) + fmt.Println("AreaZone: ", info.AreaCode) + fmt.Println("CardType: ", info.CardTypeID.ToName().String()) + fmt.Println("City: ", info.CityName) + fmt.Println("ZipCode: ", info.ZipCode) + fmt.Println("Province: ", info.ProvinceName) + return nil + } +} diff --git a/go.mod b/go.mod index 48db9d0..2847f1a 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/xluohome/phonedata go 1.16 + +require github.com/stretchr/testify v1.8.3 diff --git a/phonedatatool/model.go b/phonedatatool/model.go new file mode 100644 index 0000000..9004618 --- /dev/null +++ b/phonedatatool/model.go @@ -0,0 +1,49 @@ +package phonedatatool + +import ( + "github.com/xluohome/phonedata" + "strconv" +) + +type PhoneNumber string // 手机号码 +type AreaCode string // 区号 +func (ac AreaCode) String() string { + return string(ac) +} + +type CardTypeID byte // 卡类型 ID，单字节 + +func (ctid CardTypeID) Bytes() []byte { + return []byte{byte(ctid)} +} + +func (ctid CardTypeID) String() string { + return strconv.Itoa(int(ctid)) +} +func (ctid CardTypeID) ToName() CardTypeName { + if v, ok := phonedata.CardTypemap[byte(ctid)]; ok { + return CardTypeName(v) + } else { + return "---" + } +} + +type CardTypeName string // 卡类型中文名 +func (n CardTypeName) String() string { + return string(n) +} + +type CityName string // 城市名 +func (cn CityName) String() string { + return string(cn) +} + +type ProvinceName string // 省名 +func (pn ProvinceName) String() string { + return string(pn) +} + +type ZipCode string // 邮政编码 +func (zc ZipCode) String() string { + return string(zc) +} diff --git a/phonedatatool/pack/index.go b/phonedatatool/pack/index.go new file mode 100644 index 0000000..748a7de --- /dev/null +++ b/phonedatatool/pack/index.go @@ -0,0 +1,156 @@ +package pack + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "github.com/xluohome/phonedata/phonedatatool/util" + "sort" + "strconv" + "strings" +) + +type NumberPrefix int32 + +func (p NumberPrefix) Bytes() []byte { + return binary.LittleEndian.AppendUint32(nil, uint32(p)) +} +func (p NumberPrefix) String() string { + return strconv.Itoa(int(p)) +} + +type NumberPrefixList []NumberPrefix + +func (pl NumberPrefixList) Len() int { + return len(pl) +} +func (pl NumberPrefixList) Less(i, j int) bool { + return pl[i] < pl[j] +} +func (pl NumberPrefixList) Swap(i, j int) { + pl[i], pl[j] = pl[j], pl[i] +} + +type IndexItem struct { + numberPrefix NumberPrefix + recordOffset Offset + cardTypeID phonedatatool.CardTypeID +} + +func (ii IndexItem) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.Write(ii.numberPrefix.Bytes()) + w.Write(ii.recordOffset.Bytes()) + w.Write(ii.cardTypeID.Bytes()) + return w.Bytes() +} +func (ii *IndexItem) Parse(reader *bytes.Reader) error { + buf := make([]byte, 9) + if _, err := reader.Read(buf); err != nil { + return err + } + ii.numberPrefix = NumberPrefix(binary.LittleEndian.Uint32(buf[:4])) + ii.recordOffset = Offset(binary.LittleEndian.Uint32(buf[4:8])) + ii.cardTypeID = phonedatatool.CardTypeID(buf[8]) + return nil +} + +type IndexPart struct { + prefix2item map[NumberPrefix]*IndexItem +} + +func NewIndexPart() *IndexPart { + return &IndexPart{ + prefix2item: make(map[NumberPrefix]*IndexItem), + } +} + +func (p *IndexPart) ParsePlainText(reader *bytes.Reader, id2offset map[RecordID]Offset) error { + for reader.Len() > 0 { + var words []string + if buf, err := util.ReadUntil(reader, '\n'); err != nil { + return err + } else { + words = strings.Split(string(buf), "|") + } + + if len(words) != 3 { + return fmt.Errorf("expect words len is 3, got %v, %v", len(words), words) + } + + var numberPrefix NumberPrefix + if v, err := strconv.Atoi(words[0]); err != nil { + return fmt.Errorf("invalid number prefix %v: %v", words[0], err) + } else { + numberPrefix = NumberPrefix(v) + } + + var recordOffset Offset + if v, err := strconv.Atoi(words[1]); err != nil { + return fmt.Errorf("invalid record id %v: %v", words[1], err) + } else if offset, ok := id2offset[RecordID(v)]; !ok { + return fmt.Errorf("no offset for record id %v", v) + } else { + recordOffset = offset + } + + var cardTypeID phonedatatool.CardTypeID + if v, err := strconv.Atoi(words[2]); err != nil { + return fmt.Errorf("invalid card type id %v: %v", words[2], err) + } else { + cardTypeID = phonedatatool.CardTypeID(v) + } + + p.prefix2item[numberPrefix] = &IndexItem{ + numberPrefix: numberPrefix, + recordOffset: recordOffset, + cardTypeID: cardTypeID, + } + } + return nil +} + +func (p *IndexPart) Bytes() []byte { + var prefixList NumberPrefixList + for k := range p.prefix2item { + prefixList = append(prefixList, k) + } + sort.Sort(prefixList) + + w := bytes.NewBuffer(nil) + for _, prefix := range prefixList { + w.Write(p.prefix2item[prefix].Bytes()) + } + return w.Bytes() +} + +func (p *IndexPart) Parse(reader *bytes.Reader) error { + for reader.Len() > 0 { + item := new(IndexItem) + if err := item.Parse(reader); err != nil { + return err + } + p.prefix2item[item.numberPrefix] = item + } + return nil +} + +func (p *IndexPart) BytesPlainText(offset2id map[Offset]RecordID) []byte { + w := bytes.NewBuffer(nil) + var prefixList NumberPrefixList + for k, _ := range p.prefix2item { + prefixList = append(prefixList, k) + } + sort.Sort(prefixList) + for _, prefix := range prefixList { + item := p.prefix2item[prefix] + w.WriteString(strings.Join([]string{ + prefix.String(), + offset2id[item.recordOffset].String(), + item.cardTypeID.String(), + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} diff --git a/phonedatatool/pack/index_test.go b/phonedatatool/pack/index_test.go new file mode 100644 index 0000000..74e6b50 --- /dev/null +++ b/phonedatatool/pack/index_test.go @@ -0,0 +1,106 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestIndexPart_ParsePlainText(t *testing.T) { + indexPart := NewIndexPart() + reader := bytes.NewReader([]byte("1300000|251|2\n1300001|176|2\n1300002|1|2\n")) + id2offset := map[RecordID]Offset{ + 1: 100, + 176: 200, + 251: 300, + } + assert.NoError(t, indexPart.ParsePlainText(reader, id2offset)) + assert.Equal(t, map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 300, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 200, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 100, + cardTypeID: 2, + }, + }, indexPart.prefix2item) +} + +func TestIndexPart_Bytes(t *testing.T) { + var indexPart = &IndexPart{prefix2item: map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 0x1A4E, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 0x122C, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 0x0008, + cardTypeID: 2, + }, + }} + assert.Equal(t, []byte("\x20\xD6\x13\x00\x4E\x1A\x00\x00\x02\x21\xD6\x13\x00\x2C\x12\x00\x00\x02\x22\xD6\x13\x00\x08\x00\x00\x00\x02"), indexPart.Bytes()) +} + +func TestIndexPart_Parse(t *testing.T) { + buf := []byte("\x20\xD6\x13\x00\x4E\x1A\x00\x00\x02\x21\xD6\x13\x00\x2C\x12\x00\x00\x02\x22\xD6\x13\x00\x08\x00\x00\x00\x02") + indexPart := NewIndexPart() + assert.NoError(t, indexPart.Parse(bytes.NewReader(buf))) + assert.Equal(t, map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 0x1A4E, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 0x122C, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 0x0008, + cardTypeID: 2, + }, + }, indexPart.prefix2item) +} + +func TestIndexPart_BytesPlainText(t *testing.T) { + plainTextBuf := []byte("1300000|251|2\n1300001|176|2\n1300002|1|2\n") + indexPart := &IndexPart{prefix2item: map[NumberPrefix]*IndexItem{ + 1300000: { + numberPrefix: 1300000, + recordOffset: 300, + cardTypeID: 2, + }, + 1300001: { + numberPrefix: 1300001, + recordOffset: 200, + cardTypeID: 2, + }, + 1300002: { + numberPrefix: 1300002, + recordOffset: 100, + cardTypeID: 2, + }, + }} + offset2id := map[Offset]RecordID{ + 100: 1, + 200: 176, + 300: 251, + } + assert.Equal(t, plainTextBuf, indexPart.BytesPlainText(offset2id)) +} diff --git a/phonedatatool/pack/offset.go b/phonedatatool/pack/offset.go new file mode 100644 index 0000000..4006dd4 --- /dev/null +++ b/phonedatatool/pack/offset.go @@ -0,0 +1,21 @@ +package pack + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +type Offset int64 + +func (o Offset) Bytes() []byte { + return binary.LittleEndian.AppendUint32(nil, uint32(o)) +} +func (o *Offset) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return fmt.Errorf("failed to read: %v", err) + } + *o = Offset(binary.LittleEndian.Uint32(buf)) + return nil +} diff --git a/phonedatatool/pack/offset_test.go b/phonedatatool/pack/offset_test.go new file mode 100644 index 0000000..8bf34a4 --- /dev/null +++ b/phonedatatool/pack/offset_test.go @@ -0,0 +1,17 @@ +package pack + +import ( + "bytes" + "testing" +) +import "github.com/stretchr/testify/assert" + +func TestOffset_Bytes(t *testing.T) { + assert.Equal(t, []byte{0xd2, 0x04, 0x00, 0x00}, Offset(1234).Bytes()) +} + +func TestOffset_Parse(t *testing.T) { + offset := new(Offset) + assert.NoError(t, offset.Parse(bytes.NewReader([]byte{0xd2, 0x04, 0x00, 0x00}))) + assert.Equal(t, Offset(1234), *offset) +} diff --git a/phonedatatool/pack/pack.go b/phonedatatool/pack/pack.go new file mode 100644 index 0000000..9232ef1 --- /dev/null +++ b/phonedatatool/pack/pack.go @@ -0,0 +1,51 @@ +package pack + +import ( + "bytes" + "github.com/xluohome/phonedata/phonedatatool" +) + +type Packer struct{} + +const RecordPartBaseOffset = Offset(8) // record part 首字节偏移量 + +func NewPacker() phonedatatool.Packer { + return new(Packer) +} + +func (p *Packer) Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte) ([]byte, error) { + versionPart := new(VersionPart) + if err := versionPart.ParsePlainText(bytes.NewReader(versionPlainTextBuf)); err != nil { + return nil, err + } + versionPartBuf := versionPart.Bytes() + + recordPart := NewRecordPart() + if err := recordPart.ParsePlainText(bytes.NewReader(recordPlainTextBuf)); err != nil { + return nil, err + } + recordPartBuf, recordID2Offset := recordPart.Bytes(RecordPartBaseOffset) + + indexPartOffsetPart := RecordPartBaseOffset + Offset(len(recordPartBuf)) + + indexPart := NewIndexPart() + if err := indexPart.ParsePlainText(bytes.NewReader(indexPlainTextBuf), recordID2Offset); err != nil { + return nil, err + } + indexPartBuf := indexPart.Bytes() + + w := bytes.NewBuffer(nil) + if _, err := w.Write(versionPartBuf); err != nil { + return nil, err + } + if _, err := w.Write(indexPartOffsetPart.Bytes()); err != nil { + return nil, err + } + if _, err := w.Write(recordPartBuf); err != nil { + return nil, err + } + if _, err := w.Write(indexPartBuf); err != nil { + return nil, err + } + return w.Bytes(), nil +} diff --git a/phonedatatool/pack/query.go b/phonedatatool/pack/query.go new file mode 100644 index 0000000..384bb79 --- /dev/null +++ b/phonedatatool/pack/query.go @@ -0,0 +1,47 @@ +package pack + +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool" + "strconv" +) + +type Querier struct { +} + +func NewQuerier() *Querier { + return &Querier{} +} + +func (q *Querier) Query(phoneDataBuf []byte, number string) (*phonedatatool.QueryResult, error) { + if len(number) < 7 { + return nil, fmt.Errorf("number %v is too short", number) + } + var numberPrefix NumberPrefix + if v, err := strconv.Atoi(number[:7]); err != nil { + return nil, err + } else { + numberPrefix = NumberPrefix(v) + } + + if result, err := new(Unpacker).unpack(bytes.NewReader(phoneDataBuf)); err != nil { + return nil, err + } else { + var indexItem *IndexItem + if item, ok := result.indexPart.prefix2item[numberPrefix]; !ok { + return nil, fmt.Errorf("unknown prefix of number %v", numberPrefix) + } else { + indexItem = item + } + recordItem := result.recordPart.id2item[result.offset2id[indexItem.recordOffset]] + return &phonedatatool.QueryResult{ + PhoneNumber: phonedatatool.PhoneNumber(number), + AreaCode: phonedatatool.AreaCode(recordItem.areaCode), + CardTypeID: indexItem.cardTypeID, + CityName: phonedatatool.CityName(recordItem.city), + ZipCode: phonedatatool.ZipCode(recordItem.zipCode), + ProvinceName: phonedatatool.ProvinceName(recordItem.province), + }, nil + } +} diff --git a/phonedatatool/pack/record.go b/phonedatatool/pack/record.go new file mode 100644 index 0000000..e0506bc --- /dev/null +++ b/phonedatatool/pack/record.go @@ -0,0 +1,162 @@ +package pack + +import ( + "bytes" + "fmt" + "github.com/xluohome/phonedata/phonedatatool/util" + "sort" + "strconv" + "strings" +) + +type RecordID int64 + +func (rid RecordID) String() string { + return strconv.FormatInt(int64(rid), 10) +} + +type RecordIDList []RecordID + +func (idl RecordIDList) Len() int { + return len(idl) +} +func (idl RecordIDList) Less(i, j int) bool { + return idl[i] < idl[j] +} +func (idl RecordIDList) Swap(i, j int) { + idl[i], idl[j] = idl[j], idl[i] +} + +type RecordItem struct { + province string + city string + zipCode string + areaCode string +} + +func (ri *RecordItem) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.WriteString(strings.Join([]string{ri.province, ri.city, ri.zipCode, ri.areaCode}, "|")) + w.WriteByte(0) + return w.Bytes() +} + +// Parse 从压缩文件读取一条 RecordItem。注意 reader 必须以 '\0' 结尾。 +func (ri *RecordItem) Parse(reader *bytes.Reader) error { + if buf, err := util.ReadUntil(reader, 0); err != nil { + return fmt.Errorf("no term char for record item: %v", err) + } else { + words := strings.Split(string(buf), "|") + if len(words) != 4 { + return fmt.Errorf("invalid item bytes, %v", string(buf)) + } + ri.province = words[0] + ri.city = words[1] + ri.zipCode = words[2] + ri.areaCode = words[3] + return nil + } +} + +type RecordPart struct { + id2item map[RecordID]*RecordItem +} + +func NewRecordPart() *RecordPart { + return &RecordPart{id2item: make(map[RecordID]*RecordItem)} +} + +func (p *RecordPart) ParsePlainText(reader *bytes.Reader) error { + for reader.Len() > 0 { + + var words []string + if b, err := util.ReadUntil(reader, '\n'); err != nil { + return err + } else { + words = strings.Split(string(b), "|") + } + if len(words) != 5 { + return fmt.Errorf("invalid record line. expect 5 words (id, province, city, zipCode, areaCode), got %v words, %v", len(words), words) + } + + var recordID RecordID + if id, err := strconv.Atoi(words[0]); err != nil { + return fmt.Errorf("invalid id format, raw=%v, err=%v", words[0], err) + } else { + recordID = RecordID(id) + } + + if _, ok := p.id2item[recordID]; ok { + return fmt.Errorf("duplicate recordID %v", recordID) + } + + p.id2item[recordID] = &RecordItem{ + province: words[1], + city: words[2], + zipCode: words[3], + areaCode: words[4], + } + } + return nil +} + +func (p *RecordPart) Bytes(baseOffset Offset) ([]byte, map[RecordID]Offset) { + w := bytes.NewBuffer(nil) + id2offset := make(map[RecordID]Offset) + + var idList RecordIDList + for k := range p.id2item { + idList = append(idList, k) + } + sort.Sort(idList) + + for _, id := range idList { + id2offset[id] = baseOffset + Offset(w.Len()) + w.Write(p.id2item[id].Bytes()) + } + return w.Bytes(), id2offset +} + +func (p *RecordPart) Parse(reader *bytes.Reader, baseOffset Offset) (map[Offset]RecordID, error) { + offset2id := make(map[Offset]RecordID) + offset := baseOffset + for id := RecordID(1); reader.Len() > 0; id++ { + var itemBuf []byte + if buf, err := util.ReadUntil(reader, 0); err != nil { + return nil, err + } else { + itemBuf = buf + itemBuf = append(itemBuf, 0) + } + item := new(RecordItem) + if err := item.Parse(bytes.NewReader(itemBuf)); err != nil { + return nil, err + } + offset2id[offset] = id + p.id2item[id] = item + offset += Offset(len(itemBuf)) + } + return offset2id, nil +} + +func (p *RecordPart) BytesPlainText() []byte { + var idList RecordIDList + for k := range p.id2item { + idList = append(idList, k) + } + sort.Sort(idList) + + w := bytes.NewBuffer(nil) + for _, id := range idList { + item := p.id2item[id] + w.WriteString(strings.Join([]string{ + id.String(), + item.province, + item.city, + item.zipCode, + item.areaCode, + }, "|")) + w.WriteByte('\n') + } + return w.Bytes() +} diff --git a/phonedatatool/pack/record_test.go b/phonedatatool/pack/record_test.go new file mode 100644 index 0000000..6bab7b0 --- /dev/null +++ b/phonedatatool/pack/record_test.go @@ -0,0 +1,97 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRecordPart_ParsePlainText(t *testing.T) { + plainText := []byte("\x31\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x0A\x32\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x0A") + recordPart := NewRecordPart() + assert.NoError(t, recordPart.ParsePlainText(bytes.NewReader(plainText))) + expectedMap := map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + } + assert.Equal(t, expectedMap, recordPart.id2item) +} + +func TestRecordPart_Bytes(t *testing.T) { + recordPart := &RecordPart{id2item: map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }} + buf, id2offset := recordPart.Bytes(8) + assert.Equal(t, []byte("\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x00\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x00"), buf) + assert.Equal(t, map[RecordID]Offset{ + 1: 8, + 2: 34, + }, id2offset) +} + +func TestRecordPart_Parse(t *testing.T) { + buf := []byte("\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x00\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x00") + recordPart := NewRecordPart() + offset2id, err := recordPart.Parse(bytes.NewReader(buf), 8) + assert.NoError(t, err) + assert.Equal(t, map[Offset]RecordID{ + 8: 1, + 34: 2, + }, offset2id) + assert.Equal(t, map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }, recordPart.id2item) +} + +func TestRecordPart_BytesPlainText(t *testing.T) { + plainText := []byte("\x31\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\xB7\xA2\xE6\xB9\x96\x7C\x32\x33\x38\x30\x30\x30\x7C\x30\x35\x35\x31\x0A\x32\x7C\xE5\xAE\x89\xE5\xBE\xBD\x7C\xE5\x90\x88\xE8\x82\xA5\x7C\x32\x33\x30\x30\x30\x30\x7C\x30\x35\x35\x31\x0A") + recordPart := &RecordPart{ + id2item: map[RecordID]*RecordItem{ + 1: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\xB7\xA2\xE6\xB9\x96", + zipCode: "238000", + areaCode: "0551", + }, + 2: { + province: "\xE5\xAE\x89\xE5\xBE\xBD", + city: "\xE5\x90\x88\xE8\x82\xA5", + zipCode: "230000", + areaCode: "0551", + }, + }, + } + assert.Equal(t, plainText, recordPart.BytesPlainText()) +} diff --git a/phonedatatool/pack/unpack.go b/phonedatatool/pack/unpack.go new file mode 100644 index 0000000..094148f --- /dev/null +++ b/phonedatatool/pack/unpack.go @@ -0,0 +1,66 @@ +package pack + +import ( + "bytes" + "github.com/xluohome/phonedata/phonedatatool" +) + +type Unpacker struct { +} + +func NewUnpacker() phonedatatool.Unpacker { + return new(Unpacker) +} + +type unpackResult struct { + versionPart *VersionPart + recordPart *RecordPart + offset2id map[Offset]RecordID + indexPart *IndexPart +} + +func (u *Unpacker) Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) { + if result, err := u.unpack(bytes.NewReader(phoneDataBuf)); err != nil { + return nil, nil, nil, err + } else { + return result.versionPart.BytesPlainText(), result.recordPart.BytesPlainText(), result.indexPart.BytesPlainText(result.offset2id), nil + } +} + +func (u *Unpacker) unpack(reader *bytes.Reader) (*unpackResult, error) { + versionPart := new(VersionPart) + if err := versionPart.Parse(reader); err != nil { + return nil, err + } + + var indexPartOffset Offset + if err := indexPartOffset.Parse(reader); err != nil { + return nil, err + } + + recordBuf := make([]byte, indexPartOffset-RecordPartBaseOffset) + if _, err := reader.Read(recordBuf); err != nil { + return nil, err + } + + recordPart := NewRecordPart() + offset2id := make(map[Offset]RecordID) + if o2i, err := recordPart.Parse(bytes.NewReader(recordBuf), RecordPartBaseOffset); err != nil { + return nil, err + } else { + offset2id = o2i + } + + indexPart := NewIndexPart() + if err := indexPart.Parse(reader); err != nil { + return nil, err + } + indexPart.BytesPlainText(offset2id) + + return &unpackResult{ + versionPart: versionPart, + recordPart: recordPart, + offset2id: offset2id, + indexPart: indexPart, + }, nil +} diff --git a/phonedatatool/pack/version.go b/phonedatatool/pack/version.go new file mode 100644 index 0000000..862041a --- /dev/null +++ b/phonedatatool/pack/version.go @@ -0,0 +1,48 @@ +package pack + +import ( + "bytes" + "fmt" +) + +type VersionPart struct { + version string +} + +// ParsePlainText 从文本文件读取 +func (p *VersionPart) ParsePlainText(reader *bytes.Reader) error { + buf := make([]byte, 4) + if n, err := reader.Read(buf); err != nil { + return err + } else if n != 4 { + return fmt.Errorf("expect read 4 bytes, but read %v bytes", n) + } + p.version = string(buf) + return nil +} + +// Bytes 打包成二进制文件里的样子 +func (p *VersionPart) Bytes() []byte { + w := bytes.NewBuffer(nil) + w.WriteString(p.version) + for w.Len() < 4 { + w.WriteByte(0) + } + return w.Bytes() +} + +func (p *VersionPart) Parse(reader *bytes.Reader) error { + buf := make([]byte, 4) + if _, err := reader.Read(buf); err != nil { + return err + } + p.version = string(buf) + return nil +} + +func (p *VersionPart) BytesPlainText() []byte { + w := bytes.NewBuffer(nil) + w.Write([]byte(p.version)) + w.WriteByte('\n') + return w.Bytes() +} diff --git a/phonedatatool/pack/version_test.go b/phonedatatool/pack/version_test.go new file mode 100644 index 0000000..dd6f115 --- /dev/null +++ b/phonedatatool/pack/version_test.go @@ -0,0 +1,28 @@ +package pack + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestVersionPart_ParsePlainText(t *testing.T) { + reader := bytes.NewReader([]byte("2306\n")) + versionPart := new(VersionPart) + assert.NoError(t, versionPart.ParsePlainText(reader)) + assert.Equal(t, "2306", versionPart.version) +} + +func TestVersionPart_Bytes(t *testing.T) { + assert.Equal(t, []byte{'2', '3', '0', '6'}, (&VersionPart{version: "2306"}).Bytes()) +} + +func TestVersionPart_Parse(t *testing.T) { + versionPart := new(VersionPart) + assert.NoError(t, versionPart.Parse(bytes.NewReader([]byte{'2', '3', '0', '6'}))) + assert.Equal(t, "2306", versionPart.version) +} + +func TestVersionPart_BytesPlainText(t *testing.T) { + assert.Equal(t, []byte("2306\n"), (&VersionPart{version: "2306"}).BytesPlainText()) +} diff --git a/phonedatatool/phonedatatool.go b/phonedatatool/phonedatatool.go new file mode 100644 index 0000000..b0e1a15 --- /dev/null +++ b/phonedatatool/phonedatatool.go @@ -0,0 +1,24 @@ +package phonedatatool + +type Unpacker interface { + // Unpack 将二进制文件的内容解包成版本文件、记录文件、索引文件的内容。 + Unpack(phoneDataBuf []byte) (versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte, err error) +} + +type Packer interface { + // Pack 将版本文件、记录文件、索引文件的内容打包成二进制文件的内容。 + Pack(versionPlainTextBuf, recordPlainTextBuf, indexPlainTextBuf []byte) ([]byte, error) +} + +type QueryResult struct { + PhoneNumber PhoneNumber + AreaCode AreaCode + CardTypeID CardTypeID + CityName CityName + ZipCode ZipCode + ProvinceName ProvinceName +} + +type Querier interface { + QueryNumber(number string) (*QueryResult, error) +} diff --git a/phonedatatool/util/file.go b/phonedatatool/util/file.go new file mode 100644 index 0000000..da2ff02 --- /dev/null +++ b/phonedatatool/util/file.go @@ -0,0 +1,27 @@ +package util + +import ( + "fmt" + "os" +) + +func AssureFileNotExist(path string) error { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil + } else { + return fmt.Errorf("check file existence %v failed: %v", path, err) + } + } else { + return fmt.Errorf("file %v already exists", path) + } +} + +func AssureAllFileNotExist(paths ...string) error { + for _, p := range paths { + if err := AssureFileNotExist(p); err != nil { + return err + } + } + return nil +} diff --git a/phonedatatool/util/reader.go b/phonedatatool/util/reader.go new file mode 100644 index 0000000..c19b6e2 --- /dev/null +++ b/phonedatatool/util/reader.go @@ -0,0 +1,17 @@ +package util + +import "bytes" + +// ReadUntil 读取，直到遇到特定字符 +func ReadUntil(reader *bytes.Reader, term byte) ([]byte, error) { + var buf []byte + for { + if b, err := reader.ReadByte(); err != nil { + return nil, err + } else if b == term { + return buf, nil + } else { + buf = append(buf, b) + } + } +}