-
Notifications
You must be signed in to change notification settings - Fork 47
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f73d95d
commit ba7bf9c
Showing
15 changed files
with
2,118 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,3 +23,9 @@ build/ | |
*.pem | ||
marmot | ||
dist/ | ||
|
||
# NPM Stuff | ||
node_modules/ | ||
npm-debug.log | ||
yarn-error.log | ||
yarn-debug.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
{ | ||
"env": { | ||
"browser": true, | ||
"es2021": true, | ||
"node": true | ||
}, | ||
"extends": [ | ||
"eslint:recommended", | ||
"plugin:react/recommended", | ||
"plugin:@typescript-eslint/recommended" | ||
], | ||
"overrides": [], | ||
"parser": "@typescript-eslint/parser", | ||
"parserOptions": { | ||
"ecmaVersion": "latest", | ||
"sourceType": "module" | ||
}, | ||
"plugins": ["react", "@typescript-eslint"], | ||
"rules": { | ||
"react/prop-types": "off", | ||
"react/react-in-jsx-scope": "off" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. | ||
|
||
# dependencies | ||
/node_modules | ||
/.pnp | ||
.pnp.js | ||
|
||
# testing | ||
/coverage | ||
|
||
# next.js | ||
/.next/ | ||
/out/ | ||
|
||
# production | ||
/build | ||
|
||
# misc | ||
.DS_Store | ||
*.pem | ||
|
||
# debug | ||
npm-debug.log* | ||
yarn-debug.log* | ||
yarn-error.log* | ||
.pnpm-debug.log* | ||
|
||
# local env files | ||
.env*.local | ||
|
||
# vercel | ||
.vercel | ||
|
||
# typescript | ||
*.tsbuildinfo | ||
next-env.d.ts |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
.next | ||
node_modules |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
/* eslint-disable @typescript-eslint/no-var-requires */ | ||
/** | ||
* @type {import('next').NextConfig} | ||
*/ | ||
const nextConfig = { | ||
reactStrictMode: true, | ||
basePath: "/marmot", | ||
}; | ||
|
||
const withNextra = require("nextra")({ | ||
theme: "nextra-theme-docs", | ||
themeConfig: "./theme.config.js", | ||
// unstable_staticImage: true, | ||
}); | ||
module.exports = withNextra(nextConfig); |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
{ | ||
"name": "docs", | ||
"version": "1.0.0", | ||
"main": "index.js", | ||
"license": "MIT", | ||
"scripts": { | ||
"start": "next start", | ||
"build": "next build", | ||
"dev": "next" | ||
}, | ||
"dependencies": { | ||
"next": "^12.3.1", | ||
"nextra": "2.0.0-beta.29", | ||
"nextra-theme-docs": "2.0.0-beta.29", | ||
"react": "^18.2.0", | ||
"react-dom": "^18.2.0" | ||
}, | ||
"devDependencies": { | ||
"@types/node": "18.8.3", | ||
"@types/react": "18.0.21", | ||
"typescript": "4.8.4" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
import "nextra-theme-docs/style.css"; | ||
|
||
export default function Nextra({ Component, pageProps }) { | ||
const getLayout = Component.getLayout || ((page) => page); | ||
return getLayout(<Component {...pageProps} />); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"index": "Overview" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
# How does it work? | ||
|
||
Marmot works by using a pretty basic trick so that each process that's access database can capture changes, | ||
and then Marmot can publish them to rest of the nodes. This is how it works internally: | ||
|
||
- Each table gets a `__marmot__<table_name>_change_log` that will record a every change via triggers to | ||
change log. | ||
- Each table gets `insert`, `update`, `delete` triggers in that kicks in `AFTER` the changes have been | ||
committed to the table. These triggers record `OLD` or `NEW` values into the table. | ||
|
||
When you are running Marmot process, it's watching for changes on DB file and WAL file. Everytime there is a change | ||
Marmot scans these tables to publish them to other nodes in cluster by: | ||
|
||
- Gather all change records, and for each record calculate a consistent hash based on table name + primary keys. | ||
- Using the hash decide the primary cluster the change belongs to. | ||
- Propose the change to the calculated cluster, before marking entry to be applied to cluster. | ||
- As soon as quorum of nodes accept and confirm change log (See `SQLiteLogDB`), row changes are applied via state machine | ||
to local tables of the node and primary proposer will remove from change log table. This means every row in database | ||
due to consensus will have only one deterministic order of changes getting in cluster in case of race-conditions. | ||
- Once the order is determined for a change it's applied in an upsert or delete manner to the table. So it's quite | ||
possible that a row committed locally is overwritten or replaced later because it was not the last one | ||
in order of cluster wide commit order. | ||
|
||
## FAQ | ||
|
||
### Won’t capturing changes with triggers use more disk space? | ||
|
||
Yes it will require additional storage to old/new values from triggers. But right now that is the only way sqlite can and should allow one to capture changes. However, in a typical setting these captured changes will be picked up pretty quickly. Disk space is usually cheapest part of modern cloud, so I won’t obsess over it. | ||
|
||
### How do I do a fresh restart? | ||
|
||
Ask marmot to remove hooks and log tables by: | ||
`marmot -db-path /path/to/your/db.db -cleanup` | ||
|
||
### How would many shards should I have? | ||
|
||
It depends on your usecase and what problem you are solving for. In a typical setting you should not need more than couple of dozen shards. While read scaling won't be a problem, your write throughput will depend on your network and | ||
disk speeds (Network being the biggest culprit). |
File renamed without changes.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
const basePath = "/marmot"; | ||
|
||
/** | ||
* @type {import("nextra-theme-docs").DocsThemeConfig} | ||
*/ | ||
export default { | ||
project: { link: "https://github.com/maxpert/marmot" }, | ||
docsRepositoryBase: "https://github.com/maxpert/marmot/tree/master/docs", | ||
titleSuffix: " - Marmot", | ||
logo: ( | ||
<> | ||
<img width={50} style={{ marginRight: 5 }} src={`${basePath}/logo.png`} /> | ||
<span className="text-gray-600 font-normal hidden md:inline"> | ||
A distributed SQLite replicator built on top of NATS | ||
</span> | ||
</> | ||
), | ||
head: ( | ||
<> | ||
<meta name="msapplication-TileColor" content="#ffffff" /> | ||
<meta name="theme-color" content="#ffffff" /> | ||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> | ||
<meta httpEquiv="Content-Language" content="en" /> | ||
<meta | ||
name="description" | ||
content="Marmot: A distributed SQLite replicator built on top of NATS" | ||
/> | ||
<meta | ||
name="og:description" | ||
content="Marmot: A distributed SQLite replicator built on top of NATS" | ||
/> | ||
<meta name="twitter:card" content="summary_large_image" /> | ||
<meta name="twitter:image" content={`${basePath}/logo.png`} /> | ||
<meta name="twitter:site:domain" content="https://github.com/maxpert/marmot" /> | ||
<meta name="twitter:url" content="https://github.com/maxpert/marmot" /> | ||
<meta | ||
name="og:title" | ||
content="Marmot: A distributed SQLite replicator built on top of NATS" | ||
/> | ||
<meta name="og:image" content={`${basePath}/logo.png`} /> | ||
<meta name="apple-mobile-web-app-title" content="Marmot" /> | ||
<link rel="apple-touch-icon" sizes="180x180" href={`${basePath}/logo.png`} /> | ||
<link rel="icon" type="image/png" sizes="192x192" href={`${basePath}/logo.png`} /> | ||
<link rel="icon" type="image/png" sizes="32x32" href={`${basePath}/logo.png`} /> | ||
<link rel="icon" type="image/png" sizes="96x96" href={`${basePath}/logo.png`} /> | ||
<link rel="icon" type="image/png" sizes="16x16" href={`${basePath}/logo.png`} /> | ||
<meta name="msapplication-TileImage" content="/ms-icon-144x144.png" /> | ||
</> | ||
), | ||
navigation: true, | ||
footer: { text: <>MIT {new Date().getFullYear()} © Marmot.</> }, | ||
editLink: { text: "Edit this page on GitHub" }, | ||
unstable_faviconGlyph: "👋", | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
{ | ||
"compilerOptions": { | ||
"target": "es5", | ||
"lib": [ | ||
"dom", | ||
"dom.iterable", | ||
"esnext" | ||
], | ||
"allowJs": true, | ||
"skipLibCheck": true, | ||
"strict": false, | ||
"forceConsistentCasingInFileNames": true, | ||
"noEmit": true, | ||
"incremental": true, | ||
"esModuleInterop": true, | ||
"module": "esnext", | ||
"moduleResolution": "node", | ||
"resolveJsonModule": true, | ||
"isolatedModules": true, | ||
"jsx": "preserve" | ||
}, | ||
"include": [ | ||
"next-env.d.ts", | ||
"**/*.ts", | ||
"**/*.tsx" | ||
], | ||
"exclude": [ | ||
"node_modules" | ||
] | ||
} |
Oops, something went wrong.