diff --git a/agendas/2023/product-sync/2023-11-09-Protocol-Product-Sync.md b/agendas/2023/product-sync/2023-11-09-Protocol-Product-Sync.md new file mode 100644 index 000000000..87a872257 --- /dev/null +++ b/agendas/2023/product-sync/2023-11-09-Protocol-Product-Sync.md @@ -0,0 +1,8 @@ +Please add agenda items here. + +1. Format of this meeting + - Can we go through the updates on each of the OKR? + +2. Situation with Consensus node counts. + + diff --git a/agendas/2023/sprint-kickoff/2023-11-13-Protocol-Sprint-Kickoff.md b/agendas/2023/sprint-kickoff/2023-11-13-Protocol-Sprint-Kickoff.md new file mode 100644 index 000000000..4e29b6cee --- /dev/null +++ b/agendas/2023/sprint-kickoff/2023-11-13-Protocol-Sprint-Kickoff.md @@ -0,0 +1,271 @@ +# Team Wins 🎉 + +- safe randomness tools are deployed and live on Mainnet 🎲 + - launch page [_Flow VRF_](https://developers.flow.com/build/advanced-concepts/randomness) on Dev Portal + - [_Cadence documentation_](https://cadence-lang.org/docs/1.0/language/built-in-functions) + - [_Random Coin Toss_ as first tutorial](https://github.com/onflow/random-coin-toss) + - tentative forum post [_Secure random number generator for Flow’s smart contracts_](https://forum.flow.com/t/secure-random-number-generator-for-flow-s-smart-contracts/5110) (will be published as blog) +- Script execution working on devnet! Will enable on mainnet this sprint. +- Event streaming available for REST on mainnet. (working with 4d on fcl integration) +- Lots of AccessAPI quality of life improvements deployed to mainnet. (grpc compression, optional CCF, historic AN tx result cache) +- Community approved [Staged contract update FLIP](https://github.com/onflow/flips/pull/179) for Cadence 1.0 release +- Shared first proposal draft for cadence 1.0 release milestones with community +- Flow mainnet upgrade (spork) done +- Mainnet is no longer in the epoch fallback mode. +- DapperLabs and FlowFoundation nodes have been split. +- Public access nodes were available during the spork (except Send Transaction) +- New hosting pattern has been successfully implemented for securely & managing running Dapper Nodes +- Benchnet2 testing has validated support and pattern for running large networks +- [CI improved](https://github.com/onflow/flow-go/pull/4906) with upgraded runners, faster runs and less flaky +- Coinbase Rosetta mainnet upgrade (spork) done, no issues + +### Mainnet Uptime SLO - Last 14 days (11/30 to 11/10) + +| | Target | Current Score | Error budget used | +|:------------------------|:------:|:-------------:|:-----------------:| +| Collection Finalization | 99.9% | 99.26% | 744% | +| Block Finalization | 99.9% | 99.26% | 744% | +| Transaction Execution | 99.9% | 99.26% | 744% | +| Block Sealing | 99.9% | 99.26% | 744% | +| Access API Liveness | 99.9% | 100% | 0% | + +#### Incidents +- Mainnet network upgrade (spork). Duration: 2.5 hrs + + +### **Performance Pod Sprint Objective - Jan B** + +**Done last sprint** + +Atree Register Inlining +- Finished investigation of smoke tests failure (in the end the test failure was a false positive) +- Completed local setup for migration testing, ready to test the migration with Atree integrated with Cadence +- [Add atRoot parameter to Value.DeepRemove](https://github.com/onflow/cadence/pull/2920) + +Storehouse +- Continued breaking down Storehouse M1 implementation into PRs, ~30% of the M1 is now reviewed and merged +- [Storehouse - add PayloadToRegister](https://github.com/onflow/flow-go/pull/4955) +- [Storehouse Bootstrap - make worker count as option for indexing checkpoint](https://github.com/onflow/flow-go/pull/4884) +- [Storehouse - Add storehouse interfaces](https://github.com/onflow/flow-go/pull/4835) + +EVM support +- Continued reviewing EVM PRs + +Other Improvements & fixes +- v0.32 ONLY: [Switch to always using the 0th partition for UUID generation](https://github.com/onflow/flow-go/pull/4893) +- [Node operation improvement - Make path configs to be usable by container by default](https://github.com/onflow/flow-go/pull/4885) + +Flow Network Ugrade support +- [\[Backport\] Make path configs to be usable by container by default](https://github.com/onflow/flow-go/pull/4905) +- [\[Util\] make chunk-data-pack-dir optional for non-execution node](https://github.com/onflow/flow-go/pull/4896) +- [Update cadence to v0.42.2-patch.1](https://github.com/dapperlabs/flow-go/pull/6897) + +**This sprint** + +- Test [migration](https://github.com/onflow/flow-go/pull/4633) of [integrated solution for Atree register inlining](https://github.com/onflow/cadence/issues/2809) +- Continue implementation of [Storehouse first milestone](https://github.com/onflow/flow-go/issues/4682) (execution state on disk) +- EVM support + - Continue with PR reviews + - Start benchmarking setup + +**On Hold** + +- Plan removal of concurrent storage bottlenecks + - Transaction fee deduction + - [Cadence Type checker is not reentrant](https://dapperlabs.slack.com/archives/CG0B7CJAJ/p1684434997197079) (type comparison depends on consistent pointer used by programs cache, program cache needs to always return the same pointer to the same type) +- [Execution stack refactor - clear separation of ingestion engine and block computer](https://github.com/onflow/flow-go/issues/4077) +- [Automated Performance Tests](https://github.com/onflow/flow-go/issues/3548) + +**Active Epics** + +- [Atree register inlining](https://github.com/onflow/atree/issues/292) + +### Cadence + +### **Stable Cadence - Jan B** +Objective: long-term support release of Cadence with no expected breaking changes + +**Done last sprint** + +Features +- [v0.42 - Port adding new `revertibleRandom` function](https://github.com/onflow/cadence/pull/2910) + +Cadence 1.0 + features + - [Implement Custom Destructor Removal](https://github.com/onflow/cadence/issues/2790) + - [Interpreting for default events](https://github.com/onflow/cadence/issues/2817) + - [Type checking for default events](https://github.com/onflow/cadence/issues/2812) + - [Support for parsing default destroy events](https://github.com/onflow/cadence/issues/2799) + - [Remove support for custom destructors](https://github.com/onflow/cadence/issues/2789) + Improvements + - [Refactor resource-reference tracking](https://github.com/onflow/cadence/pull/2916) + + FLIPs + - [FLIP 95: proposal for entitlements migration](https://github.com/onflow/flips/issues/95) + +Security Fixes (port to public repo) +- [Fix nested resource moves](https://github.com/onflow/cadence/issues/2931) +- [v0.42 - Fix nested resource moves](https://github.com/onflow/cadence/issues/2930) +- [v0.42 - Fix AuthAccount creation](https://github.com/onflow/cadence/issues/2932) + +EVM Support +- [FVM EVM: Refactor stdlib to use new approach](https://github.com/onflow/flow-go/issues/4876) + +Updates of Downstream dependencies +- Go SDK: [Update to Cadence v0.42.3](https://github.com/onflow/flow-go-sdk/issues/499) + + +**This sprint** + +- continue support EVM on FLow initiative. +- Continuing with Stable Cadence scope / discussions + - Ongoing FLIPs: + - [FLIP for new behavior for attachments with entitlements](https://github.com/onflow/flips/pull/213) +- Continue work on Cadence 1.0 migrations. +- Continue Stable Cadence Docs update and knocking tasks off the [tech debt list](https://github.com/onflow/cadence/issues/2642) +- Continue work on Cadence 1.0 release plan + +**On Hold** +- Discussion of the re-entrancy edge cases + +**Active Epics** +- [Stable Cadence (aka Cadence 1.0)](https://github.com/onflow/cadence/issues/2642) + + +### **Access & Data Availability - Peter A** +Objective: Make execution data and script execution available on Edge nodes. + +**Done last sprint** + +Script Execution: + +- [Ledger] Add special handling for global register keys - [PR 4942](https://github.com/onflow/flow-go/pull/4942) +- [Execution] Return OutOfRange instead of Internal when account block is not cached - [PR 4917](https://github.com/onflow/flow-go/pull/4917) +- [Access] Allow get blocks script calls - [PR 4894](https://github.com/onflow/flow-go/pull/4894) +- [Access] Script execution coded errors - [PR 4895](https://github.com/onflow/flow-go/pull/4895) +- [Access] Get account bugfix with tests - [PR 4862](https://github.com/onflow/flow-go/pull/4862) +- [Access] Validate addresses match network in rest api - [PR 4930](https://github.com/onflow/flow-go/pull/4930) +- [Access] Add metrics for script exec failure from missing data - [PR 4907](https://github.com/onflow/flow-go/pull/4907) +- [Access] Improve logging and validation in local script exec - [PR 4920](https://github.com/onflow/flow-go/pull/4920) +- [Access] Improve script exec compare logging - [PR 4936](https://github.com/onflow/flow-go/pull/4936) +- [Access] Cleanup script execution comparisons - [PR 4956](https://github.com/onflow/flow-go/pull/4956) + +Access API: + +- [Access] Allow all origins by default on websockets connections - [PR 4954](https://github.com/onflow/flow-go/pull/4954) +- [Flow-Go-SDK] Use CCF encoding when requesting events from AccessAPI - [PR 501](https://github.com/onflow/flow-go-sdk/pull/501) + +Misc: + +- [Access] Add wait in integration tests for index to be synced - [PR 4902](https://github.com/onflow/flow-go/pull/4902) +- [Collection] Make QC Voter more resiliant to access node instability - [PR 4924](https://github.com/onflow/flow-go/pull/4924) + +**This sprint** + +- Script Execution on ANs + - Add GetRegisters API endpoint to ExecutionData API - [Issue 4756](https://github.com/onflow/flow-go/issues/4756) + - Analyze performance issues observed on devnet - [Issue 4953](https://github.com/onflow/flow-go/issues/4953) + - Deploy to mainnet (in comparison mode) and continue analyzing results and performance issues as they come up. +- Misc + - Work with 4d on getting event streaming and CCF into libraries. + - Validate new features (historic result cache, compression, etc) + +**Active Epics** + +- Script Execution on Access Node - [Issue 4637](https://github.com/onflow/flow-go/issues/4637) +- Integrate local execution state indexes into Access API - [Issue 4750](https://github.com/onflow/flow-go/issues/4750) + + +### **Permissionless Network - Yahya H** + +**Done last sprint** +- [Balanced the inbound and outbound resource limits with backpressure](https://github.com/dapperlabs/flow-go/issues/6896) [PR4929](https://github.com/onflow/flow-go/pull/4929) +- [Discovered the root cause of AN-LN peer blocking issue on mainnet23](https://github.com/dapperlabs/flow-go/issues/6895) + - [Short-term fix deployed](https://github.com/onflow/flow-go/pull/4915) + - [Long-term fix is halfway in progress- Part 1 under review](https://github.com/onflow/flow-go/pull/4951) + + +**Ongoing (last and next sprint)** +- [Part-2 and -3 of the long term fix for AN-LN peer blocking issue on mainnet23](https://github.com/dapperlabs/flow-go/issues/6895) +- [Optimizing memory-intensive RPC inspection operations](https://github.com/dapperlabs/flow-go/issues/6870) + - [Optimizing Subscription Validator Memory Usage](https://github.com/onflow/flow-go/pull/4988) + - Optimizing RPC ID Computation (Not Started) + - Cache-based Application-Specific Score (Not Started) +- [Gossip scoring to support additional cluster prefixed control messages](https://github.com/dapperlabs/flow-internal/issues/1889) [PR4857](https://github.com/onflow/flow-go/pull/4857) +- [Implement Specific Decay per Peer ID in GossipSubSpamRecord for Improved Spam Mitigation](https://github.com/dapperlabs/flow-go/issues/6662) +- [Apply Penalty to Misbehaving Peers Based on Count and Err in InvCtrlMsgNotif](https://github.com/dapperlabs/flow-go/issues/6664) [PR4978](https://github.com/onflow/flow-go/pull/4978) +- [[CI][Testing] Increase GitHub CI runners for resource intensive tests](https://github.com/dapperlabs/flow-go/issues/6894) [PR4906](https://github.com/onflow/flow-go/pull/4906) + +**Next Sprint** +- [GossipSub Message Replay Attack Mitigation](https://github.com/dapperlabs/flow-go/issues/6809) +- [Concluding GossipSub Message Forensics FLIP and planning the development](https://github.com/onflow/flips/pull/195) +- [Implement a Configurable Silence Period to Prevent False-Positive Node Penalties at Startup](https://github.com/onflow/flow-go/issues/4979) +- [Increase test coverage for BFTune ingress unit tests](https://github.com/dapperlabs/flow-go/issues/6883) +- [[BFT Testing] Refactor Orchestrator lock contension to use worker pools](https://github.com/dapperlabs/flow-go/issues/6884) + + +**Active Epics** + +- https://github.com/dapperlabs/flow-go/issues/6287 +- https://github.com/dapperlabs/flow-go/issues/6468 +- BFT https://github.com/dapperlabs/flow-go/issues/6142 +- BFT https://github.com/dapperlabs/flow-go/issues/6398 +- BN2 https://github.com/dapperlabs/flow-go/issues/6341 +- TPS https://github.com/dapperlabs/flow-go/issues/6296 +- [Zero Quarantined Networking Layer Tests Epic](https://github.com/onflow/flow-go/issues/4816) + +### **Consensus (Dynamic Protocol State) - Alex H** + +**Done last sprint for Dynamic Protocol State** + +- multiple iterations on the framework code ([PR #4834](https://github.com/onflow/flow-go/pull/4834), [PR #4868](https://github.com/onflow/flow-go/pull/4868)) wrt modularization, maintainability, documentation +- working on code for Epoch Fallback Mode (same functionality as currently, implemented in the new framework): [PR #4931](https://github.com/onflow/flow-go/pull/4931) +- Explorations on integrating Epoch Fallback Mode: [PR #4931](https://github.com/onflow/flow-go/pull/4931) + [PR #4922](https://github.com/onflow/flow-go/pull/4922) + + noticed potential avenue for leaving Epoch Fallback Mode without spork utilizing Dynamic Protocol State + +**Done last sprint (other topics)** +- Secure Randomness [Flow Dev Portal → Flow VRF](https://developers.flow.com/build/advanced-concepts/randomness) & GH repo: [Random Coin Toss in Cadence](https://github.com/onflow/random-coin-toss), [Flow VRF: Secure Randomness for Cadence - Executive summary](https://www.notion.so/dapperlabs/Flow-VRF-Secure-Randomness-for-Cadence-Executive-summary-601d0f45680b42b7bd751d440d068aac?pvs=4) (marketing) +- Jordan working on important automation of Cruise Control system ([FLIP 204]([url](https://github.com/onflow/flow-go/issues/4948))), to run Block-Time controller on networks other than mainnet + + +**Next Sprint** +- Block payload contains Hash commitment of protocol state for child block(s) ([PR #4868](https://github.com/onflow/flow-go/pull/4868)) +- continue on Epoch Fallback Mode [PR #4931](https://github.com/onflow/flow-go/pull/4931) +- Removing dynamic weight from protocol state, as this is no longer needed +- Long list remaining tech todos [epic #4649](https://github.com/onflow/flow-go/issues/4649)) + + + +### **Infra - JP** + +**Done last sprint** +- Update Ansible automation for Dapper nodes +- Prepare monitoring, alerting, and pager dudty group for Dapper Node monitoring +- Prepare Dapper infra/keys for Mainnet spork +- Create Flow Foundation infrastructure & configuration for Mainnet spork +- Create Dapper infrastructure & configuration for Mainnet spork +- Scale down networks following s +- Assist with sporks & HCUs +- Update logrotate for Dapper nodes +- Assist with BN2 testing + +**This Sprint** + +************Node Hosting************ +- Continue to monitor system configuratiton for Dapper Nodes +- Prepare execution & protocol state archives for Mainnet23 +- Scale down Mainnet23 AN & EN +- Unstake & deprecate Dapper Nodes on Devnet49 +- Clean up unneeded DPS nodes + +************Support************ +- Evaluate migration path for GCP projects and resources +- Continue to support Benchnet2 with updates to Helm chart +- Work with DevEx team to unblock migration of workloads to + +### Key Release Dates & Breaking Changes + +- Mainnet/Testnet Spork dates + - Next spork - TBD (~Q1 2024) diff --git a/agendas/2023/updates/2023-11-Flow-Ecosystem-Updates.md b/agendas/2023/updates/2023-11-Flow-Ecosystem-Updates.md new file mode 100644 index 000000000..338f7cd5c --- /dev/null +++ b/agendas/2023/updates/2023-11-Flow-Ecosystem-Updates.md @@ -0,0 +1,27 @@ +# Flow Ecosystem Updates, November 2023 + +## Announcements and Wins! 🎉 + +## 🗒 Flow Updates: + +### Educational, learning resources, tutorials and documentation + +### Tooling, infrastructure and other software for builders + +### Protocol, core contracts and node updates + +### Cadence and smart contract updates + +### Innovation, composability and advanced concepts + +### Mainnet, Sporks + +------------------------------------------ + +## Grant project updates + +------------------------------------------ +## Community updates + +------------------------------------------ +## Other misc items and updates diff --git a/sporks.json b/sporks.json index 6f9a00d3b..713234ba1 100644 --- a/sporks.json +++ b/sporks.json @@ -1,9 +1,53 @@ { "networks": { "mainnet": { + "mainnet24": { + "id": 24, + "live": true, + "name": "Mainnet-24", + "sporkTime": "2023-11-08T16:00:00Z", + "rootHeight": "65264619", + "rootParentId": "aace2d9b6e66067989d0f71c2efff38fe30d88da61e3d80946c7e7b4ee2bbc2f", + "rootStateCommitment": "3982471046dba55a6f4d58a81db6da830e8aeb138c7a5ceba88cfe6a2e87630f", + "gitCommitHash": " da42bac0e9f29fe43c4498d0d6795eb0a197730b", + "stateArtefacts": { + "gcp": { + "rootCheckpointFile": "https://storage.googleapis.com/flow-genesis-bootstrap/mainnet-24-execution/public-root-information/root.checkpoint", + "rootProtocolStateSnapshot": "https://storage.googleapis.com/flow-genesis-bootstrap/mainnet-24-execution/public-root-information/root-protocol-state-snapshot.json", + "rootProtocolStateSnapshotSignature": "https://storage.googleapis.com/flow-genesis-bootstrap/mainnet-24-execution/public-root-information/root-protocol-state-snapshot.json.asc", + "nodeInfo": "https://storage.googleapis.com/flow-genesis-bootstrap/mainnet-24-execution/public-root-information/node-infos.pub.json", + "executionStateBucket": "flow_public_mainnet24_execution_state" + }, + "s3": { + "rootCheckpointFile": "", + "rootProtocolStateSnapshot": "", + "nodeInfo": "", + "executionStateBucket": "flow-public-mainnet24-execution-state" + } + }, + "tags": { + "flow-go": "v0.32.6-patch.2", + "flow-dps": "v0.32.6-patch.2", + "cadence": "v0.42.3-patch.2", + "docker": "v0.32.6-patch.2" + }, + "seedNodes": [ + { + "address": "access-007.mainnet24.nodes.onflow.org:3570", + "key": "28a0d9edd0de3f15866dfe4aea1560c4504fe313fc6ca3f63a63e4f98d0e295144692a58ebe7f7894349198613f65b2d960abf99ec2625e247b1c78ba5bf2eae" + }, + { + "address": "access-008.mainnet24.nodes.onflow.org:3570", + "key": "11742552d21ac93da37ccda09661792977e2ca548a3b26d05f22a51ae1d99b9b75c8a9b3b40b38206b38951e98e4d145f0010f8942fd82ddf0fb1d670202264a" + } + ], + "accessNodes": [ + "access.mainnet.nodes.onflow.org:9000" + ] + }, "mainnet23": { "id": 23, - "live": true, + "live": false, "name": "Mainnet-23", "sporkTime": "2023-06-21T16:27:13Z", "rootHeight": "55114467", @@ -33,16 +77,12 @@ }, "seedNodes": [ { - "address": "access-007.mainnet23.nodes.onflow.org:3570", - "key": "28a0d9edd0de3f15866dfe4aea1560c4504fe313fc6ca3f63a63e4f98d0e295144692a58ebe7f7894349198613f65b2d960abf99ec2625e247b1c78ba5bf2eae" - }, - { - "address": "access-008.mainnet23.nodes.onflow.org:3570", - "key": "11742552d21ac93da37ccda09661792977e2ca548a3b26d05f22a51ae1d99b9b75c8a9b3b40b38206b38951e98e4d145f0010f8942fd82ddf0fb1d670202264a" + "address": "access-001.mainnet23.nodes.onflow.org:3570", + "key": "ba530d6e593947d1dd2d7f8afcf122efac9070043ce7ffdc62b4c4be9899f9a3b7e57c4c975d484386b4c5ad25a2ede097cbd497942a759a6391ba9cf724f6d9" } ], "accessNodes": [ - "access.mainnet.nodes.onflow.org:9000" + "access-001.mainnet23.nodes.onflow.org:9000" ] }, "mainnet22": {