Merge pull request #94 from wongfei2009/enhance_doc

Enrich the doc so that Marmot looks more friendly to new user.

docs/package.json

@@ -10,10 +10,10 @@
     "dev": "next"
   },
   "dependencies": {
-    "next": "^12.3.1",
-    "nextra": "^2.0.0-beta.41",
-    "nextra-theme-docs": "^2.0.0-beta.41",
-    "octokit": "^2.0.10",
+    "next": "^14.0.3",
+    "nextra": "^2.13.2,",
+    "nextra-theme-docs": "^2.13.2",
+    "octokit": "^3.1.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0"
   },

docs/src/pages/_meta.json

@@ -1,6 +1,7 @@
 {
   "index": "Welcome",
-  "demo":  "Demo",
   "intro": "Overview",
+  "demo":  "Demo",
+  "cli": "CLI",
   "internals": "Internals"
 }

docs/src/pages/cli.mdx

@@ -0,0 +1,259 @@
+---
+title: Marmot CLI
+---
+
+# Marmot CLI
+
+## Usage
+
+You can run Marmot like this:
+
+```bash
+marmot --config /path/to/config.toml -cluster-addr localhost:4221 -cluster-peers 'nats://localhost:4222/'
+```
+
+You can clean up Marmot triggers and changelogs like this:
+
+```bash
+marmot --config /path/to/config.toml --cleanup
+```
+
+You can take snapshot and upload like this:
+
+```bash
+marmot --config /path/to/config.toml --save-snapshot
+```
+
+Minimal Config
+
+```toml
+seq_map_path="/tmp/marmot-1-sm.cbor"
+db_path="/tmp/marmot-1.db"
+node_id=1
+```
+
+## Flags
+
+- `--config`: Path to the configuration file.
+- `--cleanup`: If set, only cleanup marmot triggers and changelogs.
+- `--save-snapshot`: If set, only take snapshot and upload.
+- `--cluster-addr`: Cluster listening address.
+- `--cluster-peers`: Comma separated list of clusters.
+- `--leaf-servers`: Comma separated list of leaf servers.
+
+## Configuration
+
+The configuration file is in TOML format and contains the following sections:
+
+### Snapshot Configuration
+
+- `enabled`: Enable snapshotting.
+- `interval`: Interval for taking snapshots.
+- `store`: Type of storage for snapshots. Options are "nats", "s3", "webdav", "sftp".
+- `nats`: NATS object store configuration.
+- `s3`: S3 configuration.
+- `webdav`: WebDAV configuration.
+- `sftp`: SFTP configuration.
+
+### Replication Log Configuration
+
+- `shards`: Number of shards.
+- `max_entries`: Maximum number of entries.
+- `replicas`: Number of replicas.
+- `compress`: Enable compression.
+- `update_existing`: Update existing entries.
+
+### NATS Configuration
+
+- `urls`: NATS server URLs.
+- `subject_prefix`: Subject prefix for NATS messages.
+- `stream_prefix`: Stream prefix for NATS messages.
+- `server_config`: Server configuration file for NATS.
+- `seed_file`: Seed file for NATS.
+- `user_name`: User name for NATS.
+- `user_password`: User password for NATS.
+- `ca_file`: CA file for NATS.
+- `cert_file`: Certificate file for NATS.
+- `key_file`: Key file for NATS.
+- `bind_address`: Bind address for NATS.
+- `connect_retries`: Number of connection retries for NATS.
+- `reconnect_wait_seconds`: Wait time in seconds before reconnecting to NATS.
+
+### Logging Configuration
+
+- `verbose`: Enable verbose logging.
+- `format`: Logging format.
+
+### Prometheus Configuration
+
+- `bind`: Bind address for Prometheus.
+- `enable`: Enable Prometheus.
+- `namespace`: Namespace for Prometheus.
+- `subsystem`: Subsystem for Prometheus.
+
+
+
+## Sample Config File
+
+```toml 
+# Path to target SQLite database
+db_path="/tmp/marmot.db"
+
+# ID to uniquely identify your nodes in your cluster
+# It's recommended to always configure this
+# node_id=1
+
+# Path to persist the saved sequence map on disk for warm reboot
+# If this file is missing Marmot has to download snapshot
+# and replay all logs in order to restore database
+# seq_map_path="/tmp/seq-map.cbor"
+
+# Replication enabled/disabled (default: true)
+# This will allow process to consume incoming changes from NATS
+# replicate = true
+
+# Publishing enabled/disabled (default: true)
+# This will allow process to control publishing of local DB changes to NATS
+# publish = true
+
+# Number of maximum rows to process per change allows configuring the maximum number of rows Marmot
+# will process (scan/load in memory) before publishing to NATS (default: 512)
+# scan_max_changes = 512
+
+# Cleanup interval in milliseconds used to clean up published rows. This is done in order to reduce write
+# load on the system (default: 5000)
+# cleanup_interval = 5000
+
+# Sleep timeout in milliseconds, useful for serverless scenarios. If there is no activity within given timelimit,
+# a snapshot will be performed, and process will exit. Value of 0 means it's disabled (default: 0).
+# sleep_timeout = 15000
+
+# Polling interval in milliseconds, that will explicitly check DB for change logs. This should not be required,
+# it's only useful for broken or buggy file system watchers. Value of 0 means it's disabled (default: 0)
+# polling_interval = 0
+
+# Snapshots are used to limit log size and have a database snapshot backedup on your
+# configured blob storage (NATS for now). This helps speedier recovery or cold boot
+# nodes to come up. A Snapshot is taken every log entries are close to max_entries
+# configured in replication_log section. It's recommended to use a large value
+# for maximum entries in replication log, because SQLite can do 1000s of TPS
+# replaying a couple thousands of entries should be really quick.
+[snapshot]
+# Disabling snapshot disables both restore and save
+enabled=true
+# Storage for snapshot can be "nats" | "webdav" | "s3" (default "nats")
+store="nats"
+# Interval sets perodic interval in milliseconds after which an automatic snapshot should be saved
+# If there was a snapshot saved within interval range due to other log threshold triggers, then
+# new snapshot won't be saved (since it's within time range), a value of 0 means it's disabled.
+interval=0
+
+# When setting snapshot.store to "nats" [snapshot.nats] will be used to configure snapshotting details
+# NATS connection settings (urls etc.) will be loaded from global [nats] configurations
+[snapshot.nats]
+# Number of NATS replicas of snapshot object store (max 5). Recommended values: 2-3
+replicas=1
+# Bucket name for object store to save snapshot on.
+#bucket="custom-bucket-name"
+
+# When setting snapshot.store to "s3" [snapshot.s3] will be used to configure snapshotting details
+[snapshot.s3]
+# For S3 this will be `s3.region-code.amazonaws.com` (check your AWS Console for details).
+# For Minio this will point to the host where Minio lives
+endpoint="127.0.0.1:9000"
+
+# Directory path within bucket where snapshot is saved and restore from
+path="snapshots/marmot"
+
+# By default false but should be set to true depending upong Minio configuration, for S3 it should be
+# always true. This essentially lets you select between https and http for your hosting.
+use_ssl=false
+
+# Access key ID or Minio user name
+#access_key="marmot"
+
+# Secret Key or Minio password
+#secret="ChangeMe"
+
+# Bucket name where snapshots live
+bucket="marmot"
+
+[snapshot.webdav]
+# URL of the WebDAV server root
+url="https://<webdav_server>/<web_dav_path>?dir=/snapshots/path/for/marmot&login=<username>&secret=<password>"
+
+[snapshot.sftp]
+# URL of the SFTP server with path
+url="sftp://<user>:<password>@<sftp_server>:<port>/path/to/save/snapshot"
+
+# Change log that is published and persisted in JetStreams by Marmot.
+# Marmot auto-configures missing JetStreams when booting up for you.
+[replication_log]
+# Number of replicas per log to configure (user > 1 for failover and redundancy).
+replicas=1
+# Number of shards to divide the logs over, each JetStream and subject will be prefixed
+# by the configured `subject_prefix` and `stream_prefix` under nats
+shards=1
+# Max log entries JetStream should persist, JetStream is configured to drop older entries
+# Each JetStream is configured to persist on file.
+max_entries=1024
+# Enable log compression, uses zstd to compress logs as they are streamd to NATS
+# This is useful for DB storing large blobs that can be compressed.
+compress=true
+# Update existing stream if the configurations of JetStream don't match up with configurations
+# generated due to parameters above. Use this option carefully because changing shards,
+# or max_etries etc. might have undesired side-effects on existing running cluster
+update_existing=false
+
+
+# NATS server configurations
+[nats]
+# List of NATS server to use as boot server. Reference NATS documentation on how to pass
+# authentication credentials as part of URL. Leaving out this list empty will result
+# in embedded NATS server being started with node named `marmot-node-{node_id}`. 
+# NATS configuration can provided via `server_config` variable
+urls=[
+#    "nats://localhost:4222"
+#    "nats://<user>:<password>@<host>:<port>"
+]
+# Embedded server bind address
+bind_address="0.0.0.0:4222"
+# Embedded server config file (will only be used if URLs array is empty)
+server_config=""
+# Subject prefix used when publishing log entries, it's usually suffixed by shard number
+# to get the full subject name
+subject_prefix="marmot-change-log"
+# JetStream name prefix used for publishing log entries, it's usually suffixed by shard number
+# to get the full JetStream name
+stream_prefix="marmot-changes"
+# Seed file used for client nkey authentication
+# nk -gen user > user.seed
+# nk -inkey user.seed -pubout > user.pub
+# Set to user.seed
+# Reference https://docs.nats.io/running-a-nats-service/nats_admin/security/jwt#what-are-nkeys
+seed_file=""
+# User credentials used for plain user password authentication
+user_name=""
+user_password=""
+# Number of retries when establishing the NATS server connection (will only be used if URLs array is not empty)
+connect_retries=5
+# Wait time between NATS reconnect attempts (will only be used if URLs array is not empty)
+reconnect_wait_seconds=2
+
+[prometheus]
+# Enable/Disable prometheus telemetry collection
+enable=false
+# HTTP endpoint to expose for prometheus matrix collection
+# bind=":3010"
+# Namespace for prometheus (default: `marmot`), applies to all counters, gaugues, histograms
+# namespace=""
+# Subsystem for prometheus (default: empty), applies to all counters, gauges, histograms
+# subsystem=""
+
+# Console STDOUT configurations
+[logging]
+# Configure console logging
+verbose=true
+# "console" | "json"
+format="console"
+```