diff --git a/internal/jsonwriter/jsonwriter.go b/internal/jsonwriter/jsonwriter.go new file mode 100644 index 00000000..3a06980a --- /dev/null +++ b/internal/jsonwriter/jsonwriter.go @@ -0,0 +1,59 @@ +package jsonwriter + +import ( + "encoding/json" + "os" +) + +// WriteCloser writes objects as JSON array. It provides persistent layer for JSON value. +type WriteCloser interface { + Write(interface{}) error + Close() error +} + +type fileWriter struct { + name string + reverse bool + + // FIXME: いったん全部メモリにためるのであんまよくない + buf []interface{} +} + +func (fw *fileWriter) Write(v interface{}) error { + // XXX: 排他してないのでgoroutineからは使えない + fw.buf = append(fw.buf, v) + return nil +} + +func (fw *fileWriter) Close() error { + // FIXME: ファイルの作成が Close まで遅延している。本来なら CreateFile のタ + // イミングでやるのが好ましいが、いましばらく目を瞑る + f, err := os.Create(fw.name) + if err != nil { + return err + } + defer f.Close() + if fw.reverse { + reverse(fw.buf) + fw.reverse = false + } + err = json.NewEncoder(f).Encode(fw.buf) + if err != nil { + return err + } + fw.buf = nil + return nil +} + +// CreateFile creates a WriteCloser which implemented by file. +func CreateFile(name string, reverse bool) (WriteCloser, error) { + return &fileWriter{name: name}, nil +} + +func reverse(x []interface{}) { + for i, j := 0, len(x)-1; i < j; { + x[i], x[j] = x[j], x[i] + i++ + j-- + } +} diff --git a/internal/slackadapter/common.go b/internal/slackadapter/common.go new file mode 100644 index 00000000..bae9a93a --- /dev/null +++ b/internal/slackadapter/common.go @@ -0,0 +1,20 @@ +package slackadapter + +// Error represents error response of Slack. +type Error struct { + Ok bool `json:"ok"` + Err string `json:"error"` +} + +// Error returns error message. +func (err *Error) Error() string { + return err.Err +} + +// NextCursor is cursor for next request. +type NextCursor struct { + NextCursor Cursor `json:"next_cursor"` +} + +// Cursor is type of cursor of Slack API. +type Cursor string diff --git a/internal/slackadapter/conversations_history.go b/internal/slackadapter/conversations_history.go new file mode 100644 index 00000000..ca7636d2 --- /dev/null +++ b/internal/slackadapter/conversations_history.go @@ -0,0 +1,33 @@ +package slackadapter + +import ( + "context" + "errors" + "time" + + "github.com/vim-jp/slacklog-generator/internal/slacklog" +) + +// ConversationsHistoryParams is optional parameters for ConversationsHistory +type ConversationsHistoryParams struct { + Cursor Cursor `json:"cursor,omitempty"` + Inclusive bool `json:"inclusive,omitempty"` + Latest *time.Time `json:"latest,omitempty"` + Limit int `json:"limit,omitempty"` + Oldest *time.Time `json:"oldest,omitempty"` +} + +// ConversationsHistoryReponse is response for ConversationsHistory +type ConversationsHistoryReponse struct { + Ok bool `json:"ok"` + Messages []*slacklog.Message `json:"messages,omitempty"` + HasMore bool `json:"has_more"` + PinCount int `json:"pin_count"` + ResponseMetadata *NextCursor `json:"response_metadata"` +} + +// ConversationsHistory gets conversation messages in a channel. +func ConversationsHistory(ctx context.Context, token, channel string, params ConversationsHistoryParams) (*ConversationsHistoryReponse, error) { + // TODO: call Slack's conversations.history + return nil, errors.New("not implemented yet") +} diff --git a/internal/slackadapter/cursor_iter.go b/internal/slackadapter/cursor_iter.go new file mode 100644 index 00000000..2f77a875 --- /dev/null +++ b/internal/slackadapter/cursor_iter.go @@ -0,0 +1,35 @@ +package slackadapter + +import "context" + +// CursorIterator is requirements of IterateCursor iterates with cursor. +type CursorIterator interface { + Iterate(context.Context, Cursor) (Cursor, error) +} + +// CursorIteratorFunc is a function which implements CursorIterator. +type CursorIteratorFunc func(context.Context, Cursor) (Cursor, error) + +// Iterate is an implementation for CursorIterator. +func (fn CursorIteratorFunc) Iterate(ctx context.Context, c Cursor) (Cursor, error) { + return fn(ctx, c) +} + +// IterateCursor iterates CursorIterator until returning empty cursor. +func IterateCursor(ctx context.Context, iter CursorIterator) error { + var c Cursor + for { + err := ctx.Err() + if err != nil { + return err + } + next, err := iter.Iterate(ctx, c) + if err != nil { + return err + } + if next == Cursor("") { + return nil + } + c = next + } +} diff --git a/main.go b/main.go index b2023532..76447ebd 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "github.com/joho/godotenv" cli "github.com/urfave/cli/v2" "github.com/vim-jp/slacklog-generator/subcmd" + "github.com/vim-jp/slacklog-generator/subcmd/fetchmessages" "github.com/vim-jp/slacklog-generator/subcmd/serve" ) @@ -27,6 +28,7 @@ func main() { subcmd.DownloadFilesCommand, // "download-files" subcmd.GenerateHTMLCommand, // "generate-html" serve.Command, // "serve" + fetchmessages.NewCLICommand(), // "fetch-messages" } err = app.Run(os.Args) diff --git a/subcmd/fetchmessages/fetchmessages.go b/subcmd/fetchmessages/fetchmessages.go new file mode 100644 index 00000000..108b1672 --- /dev/null +++ b/subcmd/fetchmessages/fetchmessages.go @@ -0,0 +1,156 @@ +package fetchmessages + +import ( + "context" + "errors" + "flag" + "os" + "path/filepath" + "time" + + cli "github.com/urfave/cli/v2" + "github.com/vim-jp/slacklog-generator/internal/jsonwriter" + "github.com/vim-jp/slacklog-generator/internal/slackadapter" + "github.com/vim-jp/slacklog-generator/internal/slacklog" +) + +const dateFormat = "2006-01-02" + +func toDateString(ti time.Time) string { + return ti.Format(dateFormat) +} + +func parseDateString(s string) (time.Time, error) { + l, err := time.LoadLocation("Asia/Tokeyo") + if err != nil { + return time.Time{}, err + } + ti, err := time.ParseInLocation(dateFormat, s, l) + if err != nil { + return time.Time{}, err + } + return ti, nil +} + +// Run runs "fetch-messages" sub-command. It fetch messages of a channel by a +// day. +func Run(args []string) error { + var ( + token string + datadir string + date string + verbose bool + ) + fs := flag.NewFlagSet("fetch-messages", flag.ExitOnError) + fs.StringVar(&token, "token", os.Getenv("SLACK_TOKEN"), `slack token. can be set by SLACK_TOKEN env var`) + fs.StringVar(&datadir, "datadir", "_logdata", `directory to load/save data`) + fs.StringVar(&date, "date", toDateString(time.Now()), `target date to get`) + fs.BoolVar(&verbose, "verbose", false, "verbose log") + err := fs.Parse(args) + if err != nil { + return err + } + if token == "" { + return errors.New("SLACK_TOKEN environment variable requied") + } + return run(token, datadir, date, verbose) +} + +func run(token, datadir, date string, verbose bool) error { + oldest, err := parseDateString(date) + if err != nil { + return err + } + latest := oldest.AddDate(0, 0, 1) + + ct, err := slacklog.NewChannelTable(filepath.Join(datadir, "channels.json"), []string{"*"}) + if err != nil { + return err + } + + for _, sch := range ct.Channels { + outfile := filepath.Join(datadir, sch.ID, toDateString(oldest)+".json") + fw, err := jsonwriter.CreateFile(outfile, true) + if err != nil { + return err + } + err = slackadapter.IterateCursor(context.Background(), + slackadapter.CursorIteratorFunc(func(ctx context.Context, c slackadapter.Cursor) (slackadapter.Cursor, error) { + r, err := slackadapter.ConversationsHistory(ctx, token, sch.ID, slackadapter.ConversationsHistoryParams{ + Cursor: c, + Limit: 100, + Oldest: &oldest, + Latest: &latest, + }) + if err != nil { + return "", err + } + for _, m := range r.Messages { + err := fw.Write(m) + if err != nil { + return "", err + } + } + if m := r.ResponseMetadata; r.HasMore && m != nil { + return m.NextCursor, nil + } + // HasMore && ResponseMetadata == nil は明らかにエラーだがいま + // は握りつぶしてる + return "", nil + })) + if err != nil { + // ロールバック相当が好ましいが今はまだその時期ではない + fw.Close() + return err + } + err = fw.Close() + if err != nil { + return err + } + } + + return nil +} + +// NewCLICommand creates a cli.Command, which provides "fetch-messages" +// sub-command. +func NewCLICommand() *cli.Command { + var ( + token string + datadir string + date string + verbose bool + ) + return &cli.Command{ + Name: "fetch-messages", + Usage: "fetch messages of channel by day", + Action: func(c *cli.Context) error { + return run(token, datadir, date, verbose) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "token", + Usage: "slack token", + EnvVars: []string{"SLACK_TOKEN"}, + Destination: &token, + }, + &cli.StringFlag{ + Name: "datadir", + Usage: "directory to load/save data", + Value: "_logdata", + Destination: &datadir, + }, + &cli.StringFlag{ + Name: "date", + Usage: "target date to get", + Value: toDateString(time.Now()), + Destination: &date, + }, + &cli.BoolFlag{ + Name: "verbose", + Usage: "verbose log", + Destination: &verbose, + }, + }, + } +}