diff --git a/internal/filestore/channel.go b/internal/filestore/channel.go new file mode 100644 index 00000000..3d8c7ddd --- /dev/null +++ b/internal/filestore/channel.go @@ -0,0 +1,151 @@ +package filestore + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/vim-jp/slacklog-generator/internal/store" +) + +type channelStore struct { + dir string + + rw sync.RWMutex + channels []store.Channel + idxID map[string]int +} + +// Get gets a channel by ID. +func (cs *channelStore) Get(id string) (*store.Channel, error) { + err := cs.assureLoad() + if err != nil { + return nil, err + } + cs.rw.RLock() + defer cs.rw.RUnlock() + + x, ok := cs.idxID[id] + if !ok { + return nil, fmt.Errorf("channel not found, uknown id: id=%s", id) + } + if x < 0 || x >= len(cs.channels) { + return nil, fmt.Errorf("channel index collapsed, ask developers: id=%s", id) + } + c := cs.channels[x] + return &c, nil +} + +// Iterate enumerates all channels by callback. +// 呼び出し時点でチャンネル一覧のコピーが本イテレート専用に作成される。 +// コールバックが false を返すと store.ErrIterateAbort が返る +func (cs *channelStore) Iterate(iter store.ChannelIterator) error { + err := cs.assureLoad() + if err != nil { + return err + } + cs.rw.RLock() + channels := make([]store.Channel, len(cs.channels)) + copy(channels, cs.channels) + // FIXME: cs.idxID に入ってないのは省くべきでは? + cs.rw.RUnlock() + for i := range channels { + cont := iter.Iterate(&channels[i]) + if !cont { + return store.ErrIterateAbort + } + } + return nil +} + +// Upsert updates or inserts a channel in store. +// This returns true as 1st parameter, when a channel inserted. +func (cs *channelStore) Upsert(c store.Channel) (bool, error) { + if c.ID == "" { + return false, errors.New("empty ID is forbidden") + } + + err := cs.assureLoad() + if err != nil { + return false, err + } + cs.rw.Lock() + defer cs.rw.Unlock() + + c.Tidy() + + x, ok := cs.idxID[c.ID] + if ok { + cs.channels[x] = c + return true, nil + } + cs.idxID[c.ID] = len(cs.channels) + cs.channels = append(cs.channels, c) + return false, nil +} + +// Commit saves channels to file:channels.json. +func (cs *channelStore) Commit() error { + cs.rw.Lock() + defer cs.rw.Unlock() + if cs.channels == nil { + log.Printf("[DEBUG] no channels to commit. not load yet?") + return nil + } + + ids := make([]string, 0, len(cs.idxID)) + for id := range cs.idxID { + ids = append(ids, id) + } + sort.Strings(ids) + ca := make([]store.Channel, len(ids)) + for i, id := range ids { + ca[i] = cs.channels[cs.idxID[id]] + } + err := jsonWriteFile(cs.path(), ca) + if err != nil { + return err + } + + cs.replaceChannels(ca) + return nil +} + +// path returns path for channels.json +func (cs *channelStore) path() string { + return filepath.Join(cs.dir, "channels.json") +} + +// assureLoad assure channels.json is loaded. +func (cs *channelStore) assureLoad() error { + cs.rw.Lock() + defer cs.rw.Unlock() + if cs.channels != nil { + return nil + } + var channels []store.Channel + err := jsonReadFile(cs.path(), true, &channels) + if err != nil && !os.IsNotExist(err) { + return err + } + cs.replaceChannels(channels) + return nil +} + +func (cs *channelStore) replaceChannels(channels []store.Channel) { + if len(channels) == 0 { + cs.channels = []store.Channel{} + cs.idxID = map[string]int{} + return + } + idxID := make(map[string]int, len(channels)) + for i, c := range channels { + idxID[c.ID] = i + } + cs.channels = channels + cs.idxID = idxID +} diff --git a/internal/filestore/channel_test.go b/internal/filestore/channel_test.go new file mode 100644 index 00000000..4666b356 --- /dev/null +++ b/internal/filestore/channel_test.go @@ -0,0 +1,167 @@ +package filestore + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/vim-jp/slacklog-generator/internal/store" + "github.com/vim-jp/slacklog-generator/internal/testassert" +) + +func jsonToChannel(t *testing.T, s string) *store.Channel { + t.Helper() + var c store.Channel + err := json.Unmarshal([]byte(s), &c) + if err != nil { + t.Fatalf("failed to parse as Channel: %s", err) + } + return &c +} + +func TestChannelStore_Get(t *testing.T) { + cs := &channelStore{dir: "testdata/channel_read"} + + for _, tc := range []struct { + id string + exp string + }{ + {"CXXXX0001", `{"id":"CXXXX0001","name":"channel01"}`}, + {"CXXXX0002", `{"id":"CXXXX0002","name":"channel02"}`}, + {"CXXXX0003", `{"id":"CXXXX0003","name":"channel03"}`}, + {"CXXXX0004", `{"id":"CXXXX0004","name":"channel04"}`}, + {"CXXXX0005", `{"id":"CXXXX0005","name":"channel05"}`}, + } { + act, err := cs.Get(tc.id) + if err != nil { + t.Fatalf("failed to get(%s): %s", tc.id, err) + } + exp := jsonToChannel(t, tc.exp) + testassert.Equal(t, exp, act, "id:"+tc.id) + } + + c, err := cs.Get("CXXXX9999") + if err == nil { + t.Fatalf("should fail to get unknown ID: %+v", c) + } + if !strings.HasPrefix(err.Error(), "channel not found, ") { + t.Fatalf("unexpected error for getting unknown ID: %s", err) + } +} + +func TestChannelStore_Iterate(t *testing.T) { + cs := &channelStore{dir: "testdata/channel_read"} + + var act []*store.Channel + err := cs.Iterate(store.ChannelIterateFunc(func(c *store.Channel) bool { + act = append(act, c) + return true + })) + if err != nil { + t.Fatalf("iteration failed: %s", err) + } + exp := []*store.Channel{ + jsonToChannel(t, `{"id":"CXXXX0001","name":"channel01"}`), + jsonToChannel(t, `{"id":"CXXXX0002","name":"channel02"}`), + jsonToChannel(t, `{"id":"CXXXX0003","name":"channel03"}`), + jsonToChannel(t, `{"id":"CXXXX0004","name":"channel04"}`), + jsonToChannel(t, `{"id":"CXXXX0005","name":"channel05"}`), + } + testassert.Equal(t, exp, act, "simple iteration") +} + +func TestChannelStore_Iterate_Break(t *testing.T) { + cs := &channelStore{dir: "testdata/channel_read"} + + i := 0 + err := cs.Iterate(store.ChannelIterateFunc(func(_ *store.Channel) bool { + i++ + return i > 2 + })) + if !errors.Is(err, store.ErrIterateAbort) { + t.Fatalf("iterate should be failed with:%s got:%s", store.ErrIterateAbort, err) + } +} + +func TestChannelStore_Write(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "channel_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + cs := &channelStore{dir: dir} + + for i, s := range []string{ + `{"id":"W0001","name":"channel01"}`, + `{"id":"W0009","name":"channel09"}`, + `{"id":"W0005","name":"channel05"}`, + `{"id":"W0001","name":"channel01a"}`, + } { + _, err := cs.Upsert(*jsonToChannel(t, s)) + if err != nil { + t.Fatalf("upsert failed #%d: %s", i, err) + } + } + err = cs.Commit() + if err != nil { + t.Fatalf("commit failed: %s", err) + } + + cs2 := &channelStore{dir: dir} + err = cs2.assureLoad() + if err != nil { + t.Fatalf("assureLoad failed: %s", err) + } + testassert.Equal(t, []store.Channel{ + *jsonToChannel(t, `{"id":"W0001","name":"channel01a"}`), + *jsonToChannel(t, `{"id":"W0005","name":"channel05"}`), + *jsonToChannel(t, `{"id":"W0009","name":"channel09"}`), + }, cs2.channels, "wrote channels.json") +} + +func TestChannelStore_Upsert_NoID(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "channel_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + cs := &channelStore{dir: dir} + _, err = cs.Upsert(*jsonToChannel(t, `{"name":"foobar"}`)) + if err == nil { + t.Fatal("upsert without ID should be failed") + } + if err.Error() != "empty ID is forbidden" { + t.Fatalf("unexpected failure: %s", err) + } +} + +func TestChannelStore_Commit_Empty(t *testing.T) { + // 空のCommitは channels.json を作らない + dir, err := ioutil.TempDir("testdata", "channel_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + cs := &channelStore{dir: dir} + + err = cs.Commit() + if err != nil { + t.Fatalf("unexpted failure: %s", err) + } + fi, err := os.Stat(cs.path()) + if err == nil { + t.Fatalf("channels.json created unexpectedly: %s", fi.Name()) + } + if !os.IsNotExist(err) { + t.Fatalf("unexpected failure: %s", err) + } +} diff --git a/internal/filestore/emoji.go b/internal/filestore/emoji.go new file mode 100644 index 00000000..53e8d580 --- /dev/null +++ b/internal/filestore/emoji.go @@ -0,0 +1,153 @@ +package filestore + +import ( + "crypto/md5" + "errors" + "fmt" + "log" + "os" + "path" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/vim-jp/slacklog-generator/internal/store" +) + +type emojiItem struct { + store.Emoji + + AliasTo string `json:"alias_to,omitempty"` + Path string `json:"path,omitempty"` +} + +const aliasPrefix = "alias:" + +func newEmojiItem(src store.Emoji) emojiItem { + dst := emojiItem{Emoji: src} + if strings.HasPrefix(src.URL, aliasPrefix) { + dst.AliasTo = src.URL[len(aliasPrefix):] + } else { + d := fmt.Sprintf("%x", md5.Sum([]byte(src.URL))) + ext := path.Ext(src.URL) + dst.Path = path.Join(d[len(d)-2:], d+"."+ext) + } + return dst +} + +type emojiStore struct { + dir string + + rw sync.RWMutex + emojis []emojiItem + idxName map[string]int +} + +// Get gets an emoji by name. +func (es *emojiStore) Get(name string) (*store.Emoji, error) { + err := es.assureLoad() + if err != nil { + return nil, err + } + es.rw.RLock() + defer es.rw.RUnlock() + + x, ok := es.idxName[name] + if !ok { + return nil, fmt.Errorf("emoji not found, uknown name: name=%s", name) + } + if x < 0 || x >= len(es.emojis) { + return nil, fmt.Errorf("emoji index collapsed, ask developers: name=%s", name) + } + e := es.emojis[x] + return &e.Emoji, nil +} + +// Upsert updates or inserts a emoji in store. +// This returns true as 1st parameter, when a emoji inserted. +func (es *emojiStore) Upsert(e store.Emoji) (bool, error) { + if e.Name == "" { + return false, errors.New("empty name is forbidden") + } + + err := es.assureLoad() + if err != nil { + return false, err + } + es.rw.Lock() + defer es.rw.Unlock() + + e.Tidy() + + x, ok := es.idxName[e.Name] + if ok { + es.emojis[x] = newEmojiItem(e) + return false, nil + } + es.idxName[e.Name] = len(es.emojis) + es.emojis = append(es.emojis, newEmojiItem(e)) + return true, nil +} + +// Commit saves emojis to file:emojis.json. +func (es *emojiStore) Commit() error { + es.rw.Lock() + defer es.rw.Unlock() + if es.emojis == nil { + log.Printf("[DEBUG] no emojis to commit. not load yet?") + return nil + } + + names := make([]string, 0, len(es.idxName)) + for name := range es.idxName { + names = append(names, name) + } + sort.Strings(names) + ca := make([]emojiItem, len(names)) + for i, name := range names { + ca[i] = es.emojis[es.idxName[name]] + } + err := jsonWriteFile(es.path(), ca) + if err != nil { + return err + } + + es.replaceEmojis(ca) + return nil +} + +// path returns path for emojis.json +func (es *emojiStore) path() string { + return filepath.Join(es.dir, "emojis.json") +} + +// assureLoad assure emojis.json is loaded. +func (es *emojiStore) assureLoad() error { + es.rw.Lock() + defer es.rw.Unlock() + if es.emojis != nil { + return nil + } + var emojis []emojiItem + err := jsonReadFile(es.path(), true, &emojis) + if err != nil && !os.IsNotExist(err) { + return err + } + es.replaceEmojis(emojis) + return nil +} + +func (es *emojiStore) replaceEmojis(emojis []emojiItem) { + if len(emojis) == 0 { + es.emojis = []emojiItem{} + es.idxName = map[string]int{} + return + } + idxName := make(map[string]int, len(emojis)) + for i, e := range emojis { + idxName[e.Name] = i + } + es.emojis = emojis + es.idxName = idxName +} diff --git a/internal/filestore/emoji_test.go b/internal/filestore/emoji_test.go new file mode 100644 index 00000000..902697dc --- /dev/null +++ b/internal/filestore/emoji_test.go @@ -0,0 +1,144 @@ +package filestore + +import ( + "encoding/json" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/vim-jp/slacklog-generator/internal/store" + "github.com/vim-jp/slacklog-generator/internal/testassert" +) + +func jsonToEmoji(t *testing.T, s string) *store.Emoji { + t.Helper() + var e store.Emoji + err := json.Unmarshal([]byte(s), &e) + if err != nil { + t.Fatalf("failed to parse as Emoji: %s", err) + } + return &e +} + +func jsonToEmojiItem(t *testing.T, s string) *emojiItem { + t.Helper() + var e emojiItem + err := json.Unmarshal([]byte(s), &e) + if err != nil { + t.Fatalf("failed to parse as Emoji: %s", err) + } + return &e +} + +func TestEmojiStore_Get(t *testing.T) { + es := &emojiStore{dir: "testdata/emoji_read"} + + for _, tc := range []struct { + name string + exp string + }{ + {"emoji01", `{"name":"emoji01","url":"http://example.org/emoji/0001.png"}`}, + {"emoji02", `{"name":"emoji02","url":"http://example.org/emoji/0002.png"}`}, + {"emoji03", `{"name":"emoji03","url":"http://example.org/emoji/0003.png"}`}, + {"emoji04", `{"name":"emoji04","url":"http://example.org/emoji/0004.png"}`}, + {"emoji05", `{"name":"emoji05","url":"http://example.org/emoji/0005.png"}`}, + } { + act, err := es.Get(tc.name) + if err != nil { + t.Fatalf("failed to get(%s): %s", tc.name, err) + } + exp := jsonToEmoji(t, tc.exp) + testassert.Equal(t, exp, act, "name:"+tc.name) + } + + e, err := es.Get("emoji99") + if err == nil { + t.Fatalf("should fail to get unknown name: %+v", e) + } + if !strings.HasPrefix(err.Error(), "emoji not found, ") { + t.Fatalf("unexpected error for getting unknown name: %s", err) + } +} + +func TestEmojiStore_UpsertCommit(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "emoji_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + es := &emojiStore{dir: dir} + + for i, s := range []string{ + `{"name":"emoji01","url":"http://example.org/emoji/0001.png"}`, + `{"name":"emoji09","url":"http://example.org/emoji/0009.png"}`, + `{"name":"emoji05","url":"http://example.org/emoji/0005.png"}`, + `{"name":"emoji01","url":"http://example.org/emoji/0001a.png"}`, + `{"name":"emoji99","url":"alias:emoji09"}`, + } { + _, err := es.Upsert(*jsonToEmoji(t, s)) + if err != nil { + t.Fatalf("upsert failed #%d: %s", i, err) + } + } + err = es.Commit() + if err != nil { + t.Fatalf("commit failed: %s", err) + } + + cs2 := &emojiStore{dir: dir} + err = cs2.assureLoad() + if err != nil { + t.Fatalf("assureLoad failed: %s", err) + } + testassert.Equal(t, []emojiItem{ + *jsonToEmojiItem(t, `{"name":"emoji01","url":"http://example.org/emoji/0001a.png","path":"0e/389a8ff9cd74563efd4209753ef0760e..png"}`), + *jsonToEmojiItem(t, `{"name":"emoji05","url":"http://example.org/emoji/0005.png","path":"31/055b8e6280bf3ce59f538118895d7831..png"}`), + *jsonToEmojiItem(t, `{"name":"emoji09","url":"http://example.org/emoji/0009.png","path":"b4/a39a87aca4c8b9de99230e380d0b9ab4..png"}`), + *jsonToEmojiItem(t, `{"name":"emoji99","url":"alias:emoji09","alias_to":"emoji09"}`), + }, cs2.emojis, "wrote emojis.json") +} + +func TestEmojiStore_Upsert_NoName(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "emoji_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + cs := &emojiStore{dir: dir} + _, err = cs.Upsert(*jsonToEmoji(t, `{"url":"http://example.org/emoji/0000.png"}`)) + if err == nil { + t.Fatal("upsert without name should be failed") + } + if err.Error() != "empty name is forbidden" { + t.Fatalf("unexpected failure: %s", err) + } +} + +func TestEmojiStore_Commit_Empty(t *testing.T) { + // 空のCommitは emojis.json を作らない + dir, err := ioutil.TempDir("testdata", "emoji_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + cs := &emojiStore{dir: dir} + + err = cs.Commit() + if err != nil { + t.Fatalf("unexpted failure: %s", err) + } + fi, err := os.Stat(cs.path()) + if err == nil { + t.Fatalf("emojis.json created unexpectedly: %s", fi.Name()) + } + if !os.IsNotExist(err) { + t.Fatalf("unexpected failure: %s", err) + } +} diff --git a/internal/filestore/filestore.go b/internal/filestore/filestore.go new file mode 100644 index 00000000..42c1854b --- /dev/null +++ b/internal/filestore/filestore.go @@ -0,0 +1,49 @@ +package filestore + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/vim-jp/slacklog-generator/internal/store" +) + +// FileStore is an implementation of Store on file system. +type FileStore struct { + dir string + + cs *channelStore + us *userStore + es *emojiStore +} + +// New creates a FileStore. +func New(dir string) (*FileStore, error) { + fi, err := os.Stat(dir) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + if !fi.IsDir() { + return nil, fmt.Errorf("path is used with not directory: %s", dir) + } + } + return &FileStore{ + dir: dir, + cs: &channelStore{ + dir: filepath.Join(dir, "slacklog_data"), + }, + us: &userStore{ + dir: filepath.Join(dir, "slacklog_data"), + }, + es: &emojiStore{ + dir: filepath.Join(dir, "emoji"), + }, + }, nil +} + +// Channel returns a Channel by ID. +func (fs *FileStore) Channel(id string) (*store.Channel, error) { + return fs.cs.Get(id) +} diff --git a/internal/filestore/json.go b/internal/filestore/json.go new file mode 100644 index 00000000..87f10233 --- /dev/null +++ b/internal/filestore/json.go @@ -0,0 +1,49 @@ +package filestore + +import ( + "encoding/json" + "os" + "path/filepath" +) + +// jsonReadFile reads a file and unmarshal its contents as JSON to `dst` +// destination object. +func jsonReadFile(name string, strict bool, dst interface{}) error { + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + d := json.NewDecoder(f) + if strict { + d.DisallowUnknownFields() + } + err = d.Decode(dst) + if err != nil { + return err + } + return nil +} + +func jsonWriteFile(name string, src interface{}) error { + dir := filepath.Dir(name) + if dir != "." { + err := os.MkdirAll(dir, 0777) + if err != nil { + return err + } + } + + f, err := os.Create(name) + if err != nil { + return err + } + defer f.Close() + + e := json.NewEncoder(f) + err = e.Encode(src) + if err != nil { + return err + } + return nil +} diff --git a/internal/filestore/message.go b/internal/filestore/message.go new file mode 100644 index 00000000..a0d85d25 --- /dev/null +++ b/internal/filestore/message.go @@ -0,0 +1,269 @@ +package filestore + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/vim-jp/slacklog-generator/internal/store" +) + +type messages struct { + rw sync.RWMutex + msgs []store.Message + idx map[string]int + + sorted bool + sparse bool +} + +func (mm *messages) Merge(op *messages) { + // lock and assure sorted both. + mm.rw.Lock() + mm.sort() + op.rw.Lock() + op.sort() + op.rw.Unlock() + op.rw.RLock() + // merge sort. + a, b := mm.msgs, op.msgs + r := make([]store.Message, len(a)+len(b), 0) + for len(a) > 0 && len(b) > 0 { + if a[0].Before(b[0]) { + r = append(r, a[0]) + a = a[1:] + } else { + r = append(r, b[0]) + b = b[1:] + } + } + if len(a) > 0 { + r = append(r, a...) + } + if len(b) > 0 { + r = append(r, b...) + } + // build new index. + idx := make(map[string]int) + for i, m := range r { + idx[m.ClientMsgID] = i + } + // update this messages and unlock. + mm.msgs, mm.idx = r, idx + op.rw.RUnlock() + mm.rw.Unlock() +} + +func (mm *messages) toDense() { + if !mm.sparse || len(mm.msgs) == 0 { + return + } + tail := len(mm.msgs) - 1 + for i := 0; i < tail; { + if mm.msgs[i].Timestamp == "" { + i++ + continue + } + for i < tail { + if mm.msgs[tail].Timestamp == "" { + break + } + tail-- + } + if i == tail { + tail = i - 1 + break + } + mm.msgs[i] = mm.msgs[tail] + tail-- + } + mm.msgs = mm.msgs[:tail+1] + mm.sparse = false +} + +func (mm *messages) sort() { + mm.toDense() + if mm.sorted || len(mm.msgs) <= 1 { + return + } + sort.Slice(mm.msgs, func(i, j int) bool { + return mm.msgs[i].Before(mm.msgs[j]) + }) + mm.sorted = true +} + +func (mm *messages) Upsert(m store.Message) (bool, error) { + if m.ClientMsgID == "" { + return false, errors.New("empty ID is forbidden") + } + m.Tidy() + mm.rw.Lock() + defer mm.rw.Unlock() + mm.sorted = false + x, ok := mm.idx[m.ClientMsgID] + if ok { + mm.msgs[x] = m + return true, nil + } + if mm.idx == nil { + mm.idx = make(map[string]int) + } + mm.idx[m.ClientMsgID] = len(mm.msgs) + mm.msgs = append(mm.msgs, m) + return false, nil +} + +func (mm *messages) Delete(m store.Message) bool { + mm.rw.Lock() + defer mm.rw.Unlock() + x, ok := mm.idx[m.ClientMsgID] + if !ok { + return false + } + mm.msgs[x] = store.Message{} + delete(mm.idx, m.ClientMsgID) + mm.sparse = true + return true +} + +type messageStore struct { + dir string +} + +// Begin starts a transaction for message. +func (ms *messageStore) Begin(channelID string) (store.MessageTx, error) { + return &messageTx{ + cid: channelID, + dir: filepath.Join(ms.dir, channelID), + }, nil +} + +type messageTx struct { + cid string + dir string + + mu sync.Mutex + upserts map[store.DateKey]*messages +} + +var _ store.MessageTx = (*messageTx)(nil) + +// Upsert updates or inserts a message in store. +func (mtx *messageTx) Upsert(m store.Message) (bool, error) { + ts, err := m.TimestampTime() + if err != nil { + return false, err + } + dk := store.Time2DateKey(ts) + + mtx.mu.Lock() + mm, ok := mtx.upserts[dk] + if !ok { + mm = &messages{ + idx: map[string]int{}, + } + if mtx.upserts == nil { + mtx.upserts = make(map[store.DateKey]*messages) + } + mtx.upserts[dk] = mm + } + mtx.mu.Unlock() + + return mm.Upsert(m) +} + +// Commit persists changes in a transaction. +func (mtx *messageTx) Commit() error { + mtx.mu.Lock() + defer mtx.mu.Unlock() + if len(mtx.upserts) == 0 { + return nil + } + for dk, mm := range mtx.upserts { + orig, err := mtx.readMsgsJSONL(dk) + if err != nil { + return err + } + mm.rw.RLock() + orig.Merge(mm) + mtx.writeMsgsJSONL(dk, orig) + mm.rw.RUnlock() + } + mtx.upserts = nil + return nil +} + +// msgsFileName generate JSONL filename for store.DateKey +func (mtx *messageTx) msgsFileName(dk store.DateKey) string { + return filepath.Join(mtx.dir, dk.String()+".jsonl") +} + +// readMsgsJSONL reads messages from a JSONL file. +// JSONL is JSON Lines where defined at http://jsonlines.org/ +func (mtx *messageTx) readMsgsJSONL(dk store.DateKey) (*messages, error) { + f, err := os.Open(mtx.msgsFileName(dk)) + if err != nil { + if os.IsNotExist(err) { + return new(messages), nil + } + return nil, err + } + defer f.Close() + mm := new(messages) + d := json.NewDecoder(f) + for d.More() { + var m store.Message + err := d.Decode(&m) + if err != nil { + return nil, err + } + mm.Upsert(m) + } + return mm, nil +} + +// writeMsgsJSONL writes messags as a JOSNL file. +// JSONL is JSON Lines where defined at http://jsonlines.org/ +func (mtx *messageTx) writeMsgsJSONL(dk store.DateKey, mm *messages) error { + err := os.MkdirAll(mtx.dir, 0777) + if err != nil { + return err + } + f, err := os.Create(mtx.msgsFileName(dk)) + if err != nil { + return err + } + defer f.Close() + e := json.NewEncoder(f) + for _, m := range mm.msgs { + err := e.Encode(m) + if err != nil { + return err + } + } + return nil +} + +// Rollback discards changes in a transaction. +func (mtx *messageTx) Rollback() error { + mtx.mu.Lock() + defer mtx.mu.Unlock() + mtx.upserts = nil + return nil +} + +// Iterate iterates messages in a TimeKey. +func (mtx *messageTx) Iterate(key store.TimeKey, iter store.MessageIterator) error { + // TODO: + return nil +} + +// Count counts messages in a TimeKey. +func (mtx *messageTx) Count(key store.TimeKey) (int, error) { + // TODO: + return 0, nil +} + diff --git a/internal/filestore/testdata/channel_read/channels.json b/internal/filestore/testdata/channel_read/channels.json new file mode 100644 index 00000000..84c95d26 --- /dev/null +++ b/internal/filestore/testdata/channel_read/channels.json @@ -0,0 +1,22 @@ +[ + { + "id": "CXXXX0001", + "name": "channel01" + }, + { + "id": "CXXXX0002", + "name": "channel02" + }, + { + "id": "CXXXX0003", + "name": "channel03" + }, + { + "id": "CXXXX0004", + "name": "channel04" + }, + { + "id": "CXXXX0005", + "name": "channel05" + } +] diff --git a/internal/filestore/testdata/emoji_read/emojis.json b/internal/filestore/testdata/emoji_read/emojis.json new file mode 100644 index 00000000..17e4e6da --- /dev/null +++ b/internal/filestore/testdata/emoji_read/emojis.json @@ -0,0 +1,22 @@ +[ + { + "name": "emoji01", + "url": "http://example.org/emoji/0001.png" + }, + { + "name": "emoji02", + "url": "http://example.org/emoji/0002.png" + }, + { + "name": "emoji03", + "url": "http://example.org/emoji/0003.png" + }, + { + "name": "emoji04", + "url": "http://example.org/emoji/0004.png" + }, + { + "name": "emoji05", + "url": "http://example.org/emoji/0005.png" + } +] diff --git a/internal/filestore/testdata/user_read/users.json b/internal/filestore/testdata/user_read/users.json new file mode 100644 index 00000000..8b2995f2 --- /dev/null +++ b/internal/filestore/testdata/user_read/users.json @@ -0,0 +1,22 @@ +[ + { + "id": "UXXXX0001", + "name": "user01" + }, + { + "id": "UXXXX0002", + "name": "user02" + }, + { + "id": "UXXXX0003", + "name": "user03" + }, + { + "id": "UXXXX0004", + "name": "user04" + }, + { + "id": "UXXXX0005", + "name": "user05" + } +] diff --git a/internal/filestore/user.go b/internal/filestore/user.go new file mode 100644 index 00000000..b77b7e8d --- /dev/null +++ b/internal/filestore/user.go @@ -0,0 +1,129 @@ +package filestore + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/vim-jp/slacklog-generator/internal/store" +) + +type userStore struct { + dir string + + rw sync.RWMutex + users []store.User + idxID map[string]int +} + +// Get gets a user by ID. +func (us *userStore) Get(id string) (*store.User, error) { + err := us.assureLoad() + if err != nil { + return nil, err + } + us.rw.RLock() + defer us.rw.RUnlock() + + x, ok := us.idxID[id] + if !ok { + return nil, fmt.Errorf("user not found, uknown id: id=%s", id) + } + if x < 0 || x >= len(us.users) { + return nil, fmt.Errorf("user index collapsed, ask developers: id=%s", id) + } + u := us.users[x] + return &u, nil +} + +// Upsert updates or inserts a user in store. +// This returns true as 1st parameter, when a user inserted. +func (us *userStore) Upsert(u store.User) (bool, error) { + if u.ID == "" { + return false, errors.New("empty ID is forbidden") + } + + err := us.assureLoad() + if err != nil { + return false, err + } + us.rw.Lock() + defer us.rw.Unlock() + + u.Tidy() + + x, ok := us.idxID[u.ID] + if ok { + us.users[x] = u + return true, nil + } + us.idxID[u.ID] = len(us.users) + us.users = append(us.users, u) + return false, nil +} + +// Commit saves users to file:users.json. +func (us *userStore) Commit() error { + us.rw.Lock() + defer us.rw.Unlock() + if us.users == nil { + log.Printf("[DEBUG] no users to commit. not load yet?") + return nil + } + + ids := make([]string, 0, len(us.idxID)) + for id := range us.idxID { + ids = append(ids, id) + } + sort.Strings(ids) + ua := make([]store.User, len(ids)) + for i, id := range ids { + ua[i] = us.users[us.idxID[id]] + } + err := jsonWriteFile(us.path(), ua) + if err != nil { + return err + } + + us.replaceUsers(ua) + return nil +} + +// path returns path for users.json +func (us *userStore) path() string { + return filepath.Join(us.dir, "users.json") +} + +// assureLoad assure users.json is loaded. +func (us *userStore) assureLoad() error { + us.rw.Lock() + defer us.rw.Unlock() + if us.users != nil { + return nil + } + var users []store.User + err := jsonReadFile(us.path(), true, &users) + if err != nil && !os.IsNotExist(err) { + return err + } + us.replaceUsers(users) + return nil +} + +func (us *userStore) replaceUsers(users []store.User) { + if len(users) == 0 { + us.users = []store.User{} + us.idxID = map[string]int{} + return + } + idxID := make(map[string]int, len(users)) + for i, u := range users { + idxID[u.ID] = i + } + us.users = users + us.idxID = idxID +} diff --git a/internal/filestore/user_test.go b/internal/filestore/user_test.go new file mode 100644 index 00000000..93b987e9 --- /dev/null +++ b/internal/filestore/user_test.go @@ -0,0 +1,139 @@ +package filestore + +import ( + "encoding/json" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/slack-go/slack" + "github.com/vim-jp/slacklog-generator/internal/store" + "github.com/vim-jp/slacklog-generator/internal/testassert" +) + +func jsonToUser(t *testing.T, s string) *store.User { + t.Helper() + var u store.User + err := json.Unmarshal([]byte(s), &u) + if err != nil { + t.Fatalf("failed to parse as User: %s", err) + } + return &u +} + +var userCmpOpts = []cmp.Option{ + cmpopts.IgnoreUnexported(slack.UserProfileCustomFields{}), +} + +func TestUserStore_Get(t *testing.T) { + us := &userStore{dir: "testdata/user_read"} + + for _, tc := range []struct { + id string + exp string + }{ + {"UXXXX0001", `{"id":"UXXXX0001","name":"user01"}`}, + {"UXXXX0002", `{"id":"UXXXX0002","name":"user02"}`}, + {"UXXXX0003", `{"id":"UXXXX0003","name":"user03"}`}, + {"UXXXX0004", `{"id":"UXXXX0004","name":"user04"}`}, + {"UXXXX0005", `{"id":"UXXXX0005","name":"user05"}`}, + } { + act, err := us.Get(tc.id) + if err != nil { + t.Fatalf("failed to get(%s): %s", tc.id, err) + } + exp := jsonToUser(t, tc.exp) + testassert.Equal(t, exp, act, "id:"+tc.id, userCmpOpts...) + } + + u, err := us.Get("CXXXX9999") + if err == nil { + t.Fatalf("should fail to get unknown ID: %+v", u) + } + if !strings.HasPrefix(err.Error(), "user not found, ") { + t.Fatalf("unexpected error for getting unknown ID: %s", err) + } +} + +func TestUserStore_Write(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "user_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + us := &userStore{dir: dir} + + for i, s := range []string{ + `{"id":"U0001","name":"user01"}`, + `{"id":"U0009","name":"user09"}`, + `{"id":"U0005","name":"user05"}`, + `{"id":"U0001","name":"user01a"}`, + } { + _, err := us.Upsert(*jsonToUser(t, s)) + if err != nil { + t.Fatalf("upsert failed #%d: %s", i, err) + } + } + err = us.Commit() + if err != nil { + t.Fatalf("commit failed: %s", err) + } + + us2 := &userStore{dir: dir} + err = us2.assureLoad() + if err != nil { + t.Fatalf("assureLoad failed: %s", err) + } + testassert.Equal(t, []store.User{ + *jsonToUser(t, `{"id":"U0001","name":"user01a"}`), + *jsonToUser(t, `{"id":"U0005","name":"user05"}`), + *jsonToUser(t, `{"id":"U0009","name":"user09"}`), + }, us2.users, "wrote users.json", userCmpOpts...) +} + +func TestUserStore_Upsert_NoID(t *testing.T) { + dir, err := ioutil.TempDir("testdata", "user_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + us := &userStore{dir: dir} + _, err = us.Upsert(*jsonToUser(t, `{"name":"foobar"}`)) + if err == nil { + t.Fatal("upsert without ID should be failed") + } + if err.Error() != "empty ID is forbidden" { + t.Fatalf("unexpected failure: %s", err) + } +} + +func TestUserStore_Commit_Empty(t *testing.T) { + // 空のCommitは users.json を作らない + dir, err := ioutil.TempDir("testdata", "user_write*") + if err != nil { + t.Fatalf("failed to TempDir: %s", err) + } + t.Cleanup(func() { + os.RemoveAll(dir) + }) + us := &userStore{dir: dir} + + err = us.Commit() + if err != nil { + t.Fatalf("unexpted failure: %s", err) + } + fi, err := os.Stat(us.path()) + if err == nil { + t.Fatalf("users.json created unexpectedly: %s", fi.Name()) + } + if !os.IsNotExist(err) { + t.Fatalf("unexpected failure: %s", err) + } +} diff --git a/internal/store/datekey.go b/internal/store/datekey.go new file mode 100644 index 00000000..680a3e1f --- /dev/null +++ b/internal/store/datekey.go @@ -0,0 +1,28 @@ +package store + +import ( + "fmt" + "time" +) + +// DateKey represents a date. +type DateKey struct { + Y uint16 + M uint8 + D uint8 +} + +// Time2DateKey converts time.Time to DateKey. +func Time2DateKey(v time.Time) DateKey { + y, m, d := v.Date() + return DateKey{Y: uint16(y), M: uint8(m), D: uint8(d)} +} + +func (dk DateKey) String() string { + return fmt.Sprintf("%04d-%02d-%02d", dk.Y, dk.M, dk.D) +} + +// Include checks a time `v` is in DateKey. +func (dk DateKey) Include(v time.Time) bool { + return Time2DateKey(v) == dk +} diff --git a/internal/store/message.go b/internal/store/message.go new file mode 100644 index 00000000..c64afbe5 --- /dev/null +++ b/internal/store/message.go @@ -0,0 +1,49 @@ +package store + +import ( + "time" + + "github.com/slack-go/slack" +) + +// Message represents message object in Slack. +type Message slack.Message + +// TimestampTime converts "Timestamp" to `time.Time`. +func (m Message) TimestampTime() (time.Time, error) { + tv, err := TimestampToTime(m.Timestamp) + if err != nil { + return time.Time{}, err + } + return tv, nil +} + +// Tidy removes sensitive data from a message. +func (m *Message) Tidy() { + // nothing to do for now. +} + +// Before returns true when `m` is older than `b` +func (m Message) Before(b Message) bool { + ta, _ := m.TimestampTime() + tb, _ := b.TimestampTime() + return ta.Before(tb) +} + +// MessageTx defines transactional object for messages. +type MessageTx interface { + Upsert(Message) (bool, error) + + Iterate(key TimeKey, iter MessageIterator) error + + Count(key TimeKey) (int, error) + + Commit() error + + Rollback() error +} + +// MessageIterator is callback for message iteration. +type MessageIterator interface { + Iterate(*Message) bool +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 00000000..3301cc12 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,69 @@ +/* +Package store defines types which be used in storage interface and +implementations. +*/ +package store + +import ( + "errors" + + "github.com/slack-go/slack" +) + +// ErrIterateAbort is returned when iteration "break"ed. +var ErrIterateAbort = errors.New("iteration aborted") + +// Channel represents channel object in Slack. +type Channel struct { + slack.Channel + + Pins []Pin `json:"pins"` +} + +// Tidy removes sensitive data from channel. +func (c *Channel) Tidy() { + // nothing to do for now. +} + +// Pin represents a pinned message for a channel. +type Pin struct { + ID string `json:"id"` + Typ string `json:"type"` + Created int64 `json:"created"` + User string `json:"user"` + Owner string `json:"owner"` +} + +// ChannelIterator is callback for channel iteration. +type ChannelIterator interface { + Iterate(*Channel) bool +} + +// ChannelIterateFunc is a function wrapper for ChannelIterator. +type ChannelIterateFunc func(*Channel) bool + +// Iterate implements ChannelIterator. +func (fn ChannelIterateFunc) Iterate(c *Channel) bool { + return fn(c) +} + +var _ ChannelIterator = ChannelIterateFunc(nil) + +// User represents user object in Slack. +type User slack.User + +// Tidy removes sensitive data from user. +func (u *User) Tidy() { + // nothing to do for now. +} + +// Emoji represents an emoji object in Slack. +type Emoji struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// Tidy removes sensitive data from emoji. +func (e *Emoji) Tidy() { + // nothing to do for now. +} diff --git a/internal/store/timekey.go b/internal/store/timekey.go new file mode 100644 index 00000000..bfe2ce08 --- /dev/null +++ b/internal/store/timekey.go @@ -0,0 +1,50 @@ +package store + +import ( + "time" +) + +// TimeKey is key type for messages. +type TimeKey struct { + Begin time.Time + End time.Time +} + +// Include checks a time `v` is between `Begin` (inclusive) and `End` +// (exclusive) +func (tk TimeKey) Include(v time.Time) bool { + if v.Before(tk.Begin) { + return false + } + return tk.End.After(v) +} + +var defaultLocation *time.Location + +func init() { + loc, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + panic(err) + } + defaultLocation = loc +} + +// TimeKeyYM creates a TimeKey of year/month. +func TimeKeyYM(year, month int) TimeKey { + b := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, defaultLocation) + e := b.AddDate(0, 1, 0) + return TimeKey{Begin: b, End: e} +} + +// TimeKeyYMD creates a TimeKey of year/month/day. +func TimeKeyYMD(year, month, day int) TimeKey { + b := time.Date(year, time.Month(month), day, 0, 0, 0, 0, defaultLocation) + e := b.AddDate(0, 0, 1) + return TimeKey{Begin: b, End: e} +} + +// TimeKeyDate creates a `TimeKey` of `ti.Date()`. +func TimeKeyDate(ti time.Time) TimeKey { + y, m, d := ti.Date() + return TimeKeyYMD(y, int(m), d) +} diff --git a/internal/store/timestamp.go b/internal/store/timestamp.go new file mode 100644 index 00000000..3f5f2426 --- /dev/null +++ b/internal/store/timestamp.go @@ -0,0 +1,36 @@ +package store + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// Timestamp represents slack's timestamp. +type Timestamp string + +// Time converts a Timestamp to time.Time. +func (ts Timestamp) Time() (time.Time, error) { + ss := strings.SplitN(string(ts), ".", 2) + sec, err := strconv.ParseInt(ss[0], 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("syntax error in seconds part: %q", ts) + } + n := len(ss[1]) + if n > 9 { + ss[1] = ss[1][:9] + } else if n < 9 { + ss[1] = ss[1] + strings.Repeat("0", 9-n) + } + nsec, err := strconv.ParseInt(ss[1], 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("syntax error in nano seconds part: %q", ts) + } + return time.Unix(sec, nsec).In(defaultLocation), nil +} + +// TimestampToTime converts slack's `Ts` string to time.Time. +func TimestampToTime(ts string) (time.Time, error) { + return Timestamp(ts).Time() +} diff --git a/internal/store/timestamp_test.go b/internal/store/timestamp_test.go new file mode 100644 index 00000000..30b32008 --- /dev/null +++ b/internal/store/timestamp_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "testing" + "time" +) + +func parseTime(t *testing.T, s string) time.Time { + t.Helper() + v, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + t.Fatalf("parseTime failed: %s", err) + } + return v +} + +func TestTimestampToTime(t *testing.T) { + for i, tc := range []struct { + ts string + exp time.Time + }{ + {"1583688494.253200", parseTime(t, "2020-03-09T02:28:14.253200000+09:00")}, + {"1583688494.2532001234", parseTime(t, "2020-03-09T02:28:14.253200123+09:00")}, + {"1583688494.253", parseTime(t, "2020-03-09T02:28:14.253000000+09:00")}, + } { + act, err := TimestampToTime(tc.ts) + if err != nil { + t.Fatalf("TimestampToTime(%q) failed: %s", tc.ts, err) + } + if !act.Equal(tc.exp) { + t.Fatalf("unexpected result for #%d: %+v %v", i, tc.exp, act) + } + } +} + +func TestTimestampToTime_NG(t *testing.T) { + for i, tc := range []struct { + ts string + err string + }{ + {"15836X8494.253200", `syntax error in seconds part: "15836X8494.253200"`}, + {"1583688494.253X00", `syntax error in nano seconds part: "1583688494.253X00"`}, + } { + _, err := TimestampToTime(tc.ts) + if err == nil { + t.Fatalf("unexpected success #%d with %s", i, tc.err) + } + if act := err.Error(); act != tc.err { + t.Fatalf("unexpected failure for #%d\nwant=%s\n got=%s", i, tc.err, act) + } + } +} diff --git a/internal/testassert/testassert.go b/internal/testassert/testassert.go new file mode 100644 index 00000000..180a3a20 --- /dev/null +++ b/internal/testassert/testassert.go @@ -0,0 +1,17 @@ +package testassert + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +// Equal compare "expect" and "actual". It shows a message with "Fatal" when +// there are some differences. +func Equal(t *testing.T, expect, actual interface{}, msg string, opts ...cmp.Option) { + t.Helper() + d := cmp.Diff(expect, actual, opts...) + if d != "" { + t.Fatalf("not equal: %s: -want +got\n%s", msg, d) + } +}