diff --git a/src/content/docs/r2/data-catalog/deleting-data.mdx b/src/content/docs/r2/data-catalog/deleting-data.mdx index 02102dccf0f1eb1..404857fbf7fbe89 100644 --- a/src/content/docs/r2/data-catalog/deleting-data.mdx +++ b/src/content/docs/r2/data-catalog/deleting-data.mdx @@ -10,10 +10,17 @@ import { FileTree } from "~/components" import { Tabs, TabItem } from "~/components" import { InlineBadge } from "~/components"; -## Deleting data in R2 Data Catalog - Deleting data from R2 Data Catalog or any Apache Iceberg catalog requires that operations are done in a transaction through the catalog itself. Manually deleting metadata or data files directly can lead to data catalog corruption. +## Automatic table maintenance +R2 Data Catalog can automatically manage table maintenance operations such as snapshot expiration and compaction. These continuous operations help keep latency and storage costs down. + - **Snapshot expiration**: Automatically removes old snapshots. This reduces metadata overhead. Data files are not removed until orphan file removal is run. + - **Compaction**: Merges small data files into larger ones. This optimizes read performance and reduces the number of files read during queries. + + Without enabling automatic maintenance, you need to manually handle these operations. + + Learn more in the [table maintenance](/r2/data-catalog/table-maintenance/) documentation. + ## Examples of enabling automatic table maintenance in R2 Data Catalog ```bash # Enable automatic snapshot expiration for entire catalog @@ -25,9 +32,13 @@ npx wrangler r2 bucket catalog snapshot-expiration enable my-bucket \ npx wrangler r2 bucket catalog compaction enable my-bucket \ --target-size 256 ``` -More information can be found in the [table maintenance](/r2/data-catalog/table-maintenance/) and [manage catalogs](/r2/data-catalog/manage-catalogs/) documentation. +Refer to additional examples in the [manage catalogs](/r2/data-catalog/manage-catalogs/) documentation. -## Examples of deleting data from R2 Data Catalog using PySpark +## Manually deleting and removing data +You need to manually delete data for: + - Complying with data retention policies such as GDPR or CCPA. + - Selective based deletes using conditional logic. + - Removing stale or unreferenced files that R2 Data Catalog does not manage. The following are basic examples using PySpark but similar operations can be performed using other Iceberg-compatible engines. To configure PySpark, refer to our [example](/r2/data-catalog/config-examples/spark-python/) or the official [PySpark documentation](https://spark.apache.org/docs/latest/api/python/getting_started/index.html). diff --git a/src/content/docs/r2/data-catalog/manage-catalogs.mdx b/src/content/docs/r2/data-catalog/manage-catalogs.mdx index d4c2cd254c85caa..6d959cada33ae54 100644 --- a/src/content/docs/r2/data-catalog/manage-catalogs.mdx +++ b/src/content/docs/r2/data-catalog/manage-catalogs.mdx @@ -82,7 +82,11 @@ npx wrangler r2 bucket catalog disable ## Enable compaction Compaction improves query performance by combining the many small files created during data ingestion into fewer, larger files according to the set `target file size`. For more information about compaction and why it's valuable, refer to [About compaction](/r2/data-catalog/table-maintenance/). +:::note[API token permission requirements] +Table maintenance operations such as compaction and snapshot expiration requires a Cloudflare API token with both R2 storage and R2 Data Catalog read/write permissions to act as a service credential. +Refer to [Authenticate your Iceberg engine](#authenticate-your-iceberg-engine) for details on creating a token with the required permissions. +::: @@ -120,12 +124,6 @@ npx wrangler r2 bucket catalog compaction enable -:::note[API token permission requirements] -Compaction requires a Cloudflare API token with both R2 storage and R2 Data Catalog read/write permissions to act as a service credential. The compaction process uses this token to read files, combine them, and update table metadata. - -Refer to [Authenticate your Iceberg engine](#authenticate-your-iceberg-engine) for details on creating a token with the required permissions. -::: - Once enabled, compaction applies retroactively to all existing tables (for catalog-level compaction) or the specified table (for table-level compaction). During open beta, we currently compact up to 2 GB worth of files once per hour for each table. ## Disable compaction @@ -165,6 +163,10 @@ npx wrangler r2 bucket catalog compaction disable +``` + +**Parameters**: +- `path` - Absolute path or relative to `/workspace` (e.g., `/app/src` or `src`) +- `options` (optional): + - `recursive` - Watch subdirectories recursively (default: `true`) + - `include` - Glob patterns to include (e.g., `['*.ts', '*.js']`) + - `exclude` - Glob patterns to exclude (default: `['.git', 'node_modules', '.DS_Store']`) + - `signal` - AbortSignal to cancel the watch + - `onEvent` - Callback for file change events + - `onError` - Callback for watch errors + +**Returns**: `WatchHandle` with `stop()` method and metadata properties + + +``` +// Watch entire project directory +const watcher = await sandbox.watch('/workspace', { + onEvent: (event) => { + console.log(`${event.type}: ${event.path}`); + console.log(`Is directory: ${event.isDirectory}`); + }, + onError: (error) => { + console.error('Watch error:', error.message); + } +}); + +// Stop watching when done +await watcher.stop(); +``` + + + +``` +// Watch specific file types in a directory +const watcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.tsx'], + exclude: ['*.test.ts', 'dist'], + onEvent: (event) => { + if (event.type === 'modify') { + console.log(`TypeScript file modified: ${event.path}`); + } + } +}); +``` + + +### `watchStream()` + +Get raw SSE stream for file watching (advanced usage). + +```ts +const stream = await sandbox.watchStream(path: string, options?: WatchRequest): Promise> +``` + +Most users should use `watch()` instead, which provides a higher-level API with proper lifecycle management. + +### `stopWatch()` + +Stop a specific watch by ID. + +```ts +await sandbox.stopWatch(watchId: string): Promise<{ success: boolean }> +``` + +**Parameters**: +- `watchId` - Watch ID from the WatchHandle + +### `listWatches()` + +List all active watches. + +```ts +const result = await sandbox.listWatches(): Promise +``` + +**Returns**: +```ts +interface WatchListResult { + success: boolean; + watches: Array<{ + id: string; + path: string; + startedAt: string; + }>; + count: number; + timestamp: string; +} +``` + +## Types + +### `WatchHandle` + +Handle returned from `watch()` to control and inspect the watch. + +```ts +interface WatchHandle { + /** Stop watching and clean up resources */ + stop(): Promise; + /** The watch ID (for debugging) */ + readonly id: string; + /** The path being watched */ + readonly path: string; +} +} +``` + +### `WatchEvent` + +File system change event passed to the `onEvent` callback. + +```ts +interface WatchEvent { + /** The type of change that occurred */ + type: WatchEventType; + /** Absolute path to the file or directory that changed */ + path: string; + /** Whether the changed path is a directory */ + isDirectory: boolean; +} +``` + +### `WatchEventType` + +Types of filesystem changes that can be detected. + +```ts +type WatchEventType = 'create' | 'modify' | 'delete' | 'rename'; +``` + +- **`create`** - File or directory was created +- **`modify`** - File content or directory attributes changed +- **`delete`** - File or directory was deleted +- **`rename`** - File or directory was moved/renamed + +### `WatchOptions` + +Configuration options for watching directories. + +```ts +interface WatchOptions { + /** Watch subdirectories recursively (default: true) */ + recursive?: boolean; + /** Glob patterns to include (e.g., ['*.ts', '*.js']) */ + include?: string[]; + /** Glob patterns to exclude (default: ['.git', 'node_modules', '.DS_Store']) */ + exclude?: string[]; + /** AbortSignal to cancel the watch */ + signal?: AbortSignal; + /** Callback for file change events */ + onEvent?: (event: WatchEvent) => void; + /** Callback for errors (e.g., watch process died) */ + onError?: (error: Error) => void; +} +``` + +## Use Cases + +### Development Server with Auto-reload + + +``` +// Watch source files and restart server on changes +const watcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.js'], + onEvent: async (event) => { + if (event.type === 'modify') { + console.log('Restarting server...'); + await sandbox.exec('pkill node'); // Stop existing server + await sandbox.exec('npm start &'); // Start new server in background + } + } +}); +``` + + +### Build System Integration + + +``` +// Trigger builds when source files change +const watcher = await sandbox.watch('/workspace/src', { + exclude: ['dist', '*.log'], + onEvent: async (event) => { + if (event.type === 'modify' && event.path.endsWith('.ts')) { + console.log('Running TypeScript build...'); + const result = await sandbox.exec('npm run build'); + if (!result.success) { + console.error('Build failed:', result.stderr); + } + } + } +}); +``` + + +### File Sync Monitoring + + +``` +// Monitor file changes for sync operations +const changedFiles = new Set(); + +const watcher = await sandbox.watch('/workspace/data', { + onEvent: (event) => { + if (event.type !== 'delete') { + changedFiles.add(event.path); + } + } +}); + +// Periodically sync changed files +setInterval(async () => { + if (changedFiles.size > 0) { + console.log(`Syncing ${changedFiles.size} changed files...`); + // Sync logic here + changedFiles.clear(); + } +}, 5000); +``` + + +## Best Practices + +### Resource Management + +Always stop watchers when done to prevent resource leaks: + + +``` +const watcher = await sandbox.watch('/workspace/src'); + +try { + // Your application logic +} finally { + await watcher.stop(); // Clean up resources +} +``` + + +### Using AbortSignal + +Use AbortSignal for graceful cancellation: + + +``` +const controller = new AbortController(); + +const watcher = await sandbox.watch('/workspace/src', { + signal: controller.signal, + onEvent: (event) => console.log(event) +}); + +// Cancel after 60 seconds +setTimeout(() => controller.abort(), 60000); +``` + + +### Filtering Events Efficiently + +Use `include` and `exclude` options rather than filtering in callbacks: + + +``` +// Efficient - filtering at inotify level +const watcher = await sandbox.watch('/workspace', { + include: ['*.ts', '*.js'], + exclude: ['node_modules', 'dist'] +}); + +// Less efficient - filtering in JavaScript +const watcher = await sandbox.watch('/workspace', { + onEvent: (event) => { + if (!event.path.endsWith('.ts') && !event.path.endsWith('.js')) { + return; // Skip non-JS/TS files + } + // Handle event + } +}); +``` + + +## Technical Details + +The file watching system uses Linux's native `inotify` via the `inotifywait` command for efficient, real-time monitoring: + +- **Performance**: Native kernel events, no polling overhead +- **Reliability**: Handles high-frequency file changes without missing events +- **Filtering**: Server-side pattern matching reduces network traffic +- **Default exclusions**: Common directories (`.git`, `node_modules`) excluded by default +- **Event coalescing**: Multiple rapid changes to the same file are properly handled + +:::note[Container lifecycle] +File watchers are automatically stopped when the sandbox container sleeps or is destroyed. You do not need to manually stop them on container shutdown. +::: + +:::caution[Path requirements] +All paths must exist when starting a watch. Watching non-existent paths returns an error. Create directories before watching them. +:::} +::: \ No newline at end of file diff --git a/src/content/docs/sandbox/api/index.mdx b/src/content/docs/sandbox/api/index.mdx index 37c4472e359f17c..7f29c5209ea620c 100644 --- a/src/content/docs/sandbox/api/index.mdx +++ b/src/content/docs/sandbox/api/index.mdx @@ -35,6 +35,14 @@ The Sandbox SDK provides a comprehensive API for executing code, managing files, Read, write, and manage files in the sandbox filesystem. Includes directory operations and file metadata. + + Monitor real-time filesystem changes using native inotify. Build development tools, hot-reload systems, and responsive file processing. + + +``` + +**Parameters**: +- `path` - Directory path to watch (absolute or relative to `/workspace`) +- `options` (optional): + - `recursive` - Watch subdirectories recursively (default: `true`) + - `include` - Array of glob patterns to include (e.g., `["*.js", "*.ts"]`) + - `exclude` - Array of patterns to exclude (default: `[".git", "node_modules", ".DS_Store"]`) + - `onEvent` - Callback function for file events + - `onError` - Callback function for watch errors + - `signal` - AbortSignal to stop watching + +**Returns**: `Promise` with `stop()` method and properties + + +```typescript +// Watch all changes in src directory +const watcher = await sandbox.watch('/workspace/src', { + recursive: true, + onEvent: (event) => { + console.log(`${event.type}: ${event.path}`); + // Outputs: "create: /workspace/src/new-file.js" + }, + onError: (error) => console.error('Watch error:', error) +}); + +// With file filtering +const tsWatcher = await sandbox.watch('/workspace', { + include: ['*.ts', '*.tsx'], + exclude: ['node_modules', '.git', 'dist'], + onEvent: (event) => { + if (event.type === 'modify') { + console.log(`TypeScript file changed: ${event.path}`); + } + } +}); + +// Stop watching when done +await watcher.stop(); +``` + + +### Using AbortController + +Control the watch lifecycle with AbortController for automatic cleanup. + + +```typescript +const controller = new AbortController(); + +const watcher = await sandbox.watch('/workspace/src', { + signal: controller.signal, + onEvent: (event) => console.log(event), + onError: (error) => console.error(error) +}); + +// Stop watching after 30 seconds +setTimeout(() => controller.abort(), 30000); + +// Or stop manually +await watcher.stop(); +``` + + +## Event Types + +Watch events contain information about filesystem operations: + +```ts +interface WatchEvent { + type: 'create' | 'modify' | 'delete' | 'rename'; + path: string; // Full path of the affected file + isDirectory: boolean; // true if the path is a directory + timestamp: string; // ISO timestamp when event occurred +} +``` + +**Event Types**: +- `create` - New file or directory created +- `modify` - File content or metadata changed +- `delete` - File or directory deleted +- `rename` - File or directory moved/renamed + + +```typescript +await sandbox.watch('/workspace', { + onEvent: (event) => { + switch (event.type) { + case 'create': + if (event.isDirectory) { + console.log(`New directory: ${event.path}`); + } else { + console.log(`New file: ${event.path}`); + } + break; + case 'modify': + console.log(`File changed: ${event.path} at ${event.timestamp}`); + break; + case 'delete': + console.log(`Deleted: ${event.path}`); + break; + case 'rename': + console.log(`Moved: ${event.path}`); + break; + } + } +}); +``` + + +## WatchHandle + +The returned handle provides control over the watch operation: + +```ts +interface WatchHandle { + stop(): Promise; // Stop watching and clean up + id: string; // Unique watch identifier + path: string; // Path being watched +} +``` + + +```typescript +const watcher = await sandbox.watch('/workspace/src'); + +console.log(`Watch ID: ${watcher.id}`); +console.log(`Watching: ${watcher.path}`); + +// Stop when done +await watcher.stop(); +``` + + +## Patterns and Filtering + +Use glob patterns to control which files trigger events: + + +```typescript +// Watch only JavaScript and TypeScript files +const jsWatcher = await sandbox.watch('/workspace', { + include: ['*.js', '*.ts', '*.jsx', '*.tsx'], + onEvent: (event) => console.log(`JS/TS change: ${event.path}`) +}); + +// Watch configuration files +const configWatcher = await sandbox.watch('/workspace', { + include: ['*.json', '*.yaml', '*.toml', '.env*'], + onEvent: (event) => { + if (event.type === 'modify') { + console.log(`Config updated: ${event.path}`); + } + } +}); + +// Exclude build artifacts and dependencies +const sourceWatcher = await sandbox.watch('/workspace', { + exclude: [ + 'node_modules', '.git', 'dist', 'build', + '*.log', '.DS_Store', 'Thumbs.db' + ], + onEvent: (event) => console.log(`Source change: ${event.path}`) +}); +``` + + +## Use Cases + +File watching enables real-time development workflows: + +### Hot Reload Development Server + + +```typescript +// Monitor source files for automatic rebuild +const devWatcher = await sandbox.watch('/workspace/src', { + include: ['*.js', '*.jsx', '*.ts', '*.tsx', '*.css'], + onEvent: async (event) => { + if (event.type === 'modify' || event.type === 'create') { + console.log(`Rebuilding due to ${event.type}: ${event.path}`); + await sandbox.exec('npm run build'); + console.log('Rebuild complete'); + } + } +}); +``` + + +### Test Runner Integration + + +```typescript +// Run tests when source or test files change +const testWatcher = await sandbox.watch('/workspace', { + include: ['**/*.test.js', 'src/**/*.js'], + onEvent: async (event) => { + if (event.type !== 'delete') { + console.log(`Running tests due to change: ${event.path}`); + await sandbox.exec('npm test'); + } + } +}); +``` + + +### Configuration Monitoring + + +```typescript +// React to configuration changes +const configWatcher = await sandbox.watch('/workspace', { + include: ['package.json', '.env*', 'config/*'], + onEvent: async (event) => { + if (event.path.includes('package.json')) { + console.log('Dependencies changed, reinstalling...'); + await sandbox.exec('npm install'); + } else if (event.path.includes('.env')) { + console.log('Environment variables updated'); + } + } +}); +``` + + +## Performance Considerations + +- **inotify limits**: Linux systems have limits on the number of watches (typically 8192). Use specific `include` patterns rather than watching entire filesystems. +- **Recursive watching**: Large directory trees can consume many watch descriptors. Consider watching specific subdirectories instead. +- **Event frequency**: High-frequency file operations (like log files) can generate many events. Use debouncing in your event handlers when needed. +- **Cleanup**: Always call `stop()` on watch handles to free system resources. + + +```typescript +// Debounced event handler for high-frequency changes +let debounceTimer: NodeJS.Timeout | null = null; + +const watcher = await sandbox.watch('/workspace/logs', { + onEvent: (event) => { + // Clear previous timer + if (debounceTimer) clearTimeout(debounceTimer); + + // Set new timer + debounceTimer = setTimeout(() => { + console.log(`Log file activity: ${event.path}`); + debounceTimer = null; + }, 1000); // 1 second debounce + } +}); +``` + + +## Error Handling + +Watch operations can fail due to filesystem issues, permission problems, or system limits: + + +```typescript +try { + const watcher = await sandbox.watch('/workspace/src', { + onError: (error) => { + console.error('Watch error:', error.message); + // Handle errors like process death, permission issues + } + }); +} catch (error) { + if (error.message.includes('does not exist')) { + console.error('Directory not found:', '/workspace/src'); + } else if (error.message.includes('permission')) { + console.error('Permission denied accessing:', '/workspace/src'); + } else { + console.error('Failed to start watching:', error.message); + } +} +``` + \ No newline at end of file diff --git a/src/content/docs/sandbox/guides/file-watching.mdx b/src/content/docs/sandbox/guides/file-watching.mdx new file mode 100644 index 000000000000000..675ddeca6e0a2cf --- /dev/null +++ b/src/content/docs/sandbox/guides/file-watching.mdx @@ -0,0 +1,414 @@ +--- +title: Watch filesystem changes +pcx_content_type: how-to +sidebar: + order: 3 +description: Monitor files and directories in real-time to build responsive development tools and automation workflows. +--- + +import { TypeScriptExample } from "~/components"; + +This guide shows you how to monitor filesystem changes in real-time using the Sandbox SDK's file watching capabilities. File watching is essential for building development tools, automated workflows, and responsive applications that react to file system changes. + +## Basic file watching + +Start by watching a directory for any changes: + + +``` +const watcher = await sandbox.watch('/workspace/src', { + onEvent: (event) => { + console.log(`${event.type} event: ${event.path}`); + console.log(`Is directory: ${event.isDirectory}`); + }, + onError: (error) => { + console.error('Watch failed:', error.message); + } +}); + +// Always clean up when done +process.on('exit', () => watcher.stop()); +``` + + +The watcher will detect four types of events: +- **create** - Files or directories created +- **modify** - File content or attributes changed +- **delete** - Files or directories removed +- **rename** - Files or directories moved/renamed + +## Filter by file type + +Use `include` patterns to watch only specific file types: + + +``` +// Only watch TypeScript and JavaScript files +const watcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.tsx', '*.js', '*.jsx'], + onEvent: (event) => { + console.log(`${event.type}: ${event.path}`); + } +}); +``` + + +Common include patterns: +- `*.ts` - TypeScript files +- `*.js` - JavaScript files +- `*.json` - JSON configuration files +- `*.md` - Markdown documentation +- `package*.json` - Package files specifically + +## Exclude directories + +Use `exclude` patterns to ignore certain directories or files: + + +``` +const watcher = await sandbox.watch('/workspace', { + exclude: [ + 'node_modules', // Dependencies + 'dist', // Build output + '*.log', // Log files + '.git', // Git metadata (excluded by default) + '*.tmp' // Temporary files + ], + onEvent: (event) => { + console.log(`Change detected: ${event.path}`); + } +}); +``` + + +:::note[Default exclusions] +The following patterns are excluded by default: `.git`, `node_modules`, `.DS_Store`. You can override this by providing your own `exclude` array. +::: + +## Build responsive development tools + +### Auto-restarting development server + +Build a development server that automatically restarts when source files change: + + +``` +let serverProcess: { stop: () => Promise } | null = null; + +async function startServer() { + if (serverProcess) { + await serverProcess.stop(); + } + + console.log('Starting development server...'); + serverProcess = await sandbox.startProcess('npm run dev', { + onOutput: (stream, data) => { + console.log(`[server] ${data}`); + } + }); +} + +const watcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.js', '*.json'], + onEvent: async (event) => { + if (event.type === 'modify') { + console.log(`Detected change in ${event.path}, restarting server...`); + await startServer(); + } + } +}); + +// Initial server start +await startServer(); +``` + + +### Auto-building on changes + +Trigger builds automatically when source files are modified: + + +``` +let buildInProgress = false; + +const watcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.tsx'], + onEvent: async (event) => { + if (event.type === 'modify' && !buildInProgress) { + buildInProgress = true; + console.log('Building TypeScript project...'); + + try { + const result = await sandbox.exec('npm run build'); + if (result.success) { + console.log('Build completed successfully'); + } else { + console.error('Build failed:', result.stderr); + } + } catch (error) { + console.error('Build error:', error); + } finally { + buildInProgress = false; + } + } + } +}); +``` + + +### Live documentation updates + +Watch documentation files and rebuild docs when they change: + + +``` +const watcher = await sandbox.watch('/workspace/docs', { + include: ['*.md', '*.mdx'], + onEvent: async (event) => { + if (event.type === 'modify') { + console.log(`Documentation updated: ${event.path}`); + + // Rebuild documentation site + const result = await sandbox.exec('npm run build:docs'); + if (result.success) { + console.log('Documentation rebuilt'); + } + } + } +}); +``` + + +## Advanced patterns + +### Debounced file operations + +Avoid excessive operations by debouncing rapid file changes: + + +``` +let debounceTimeout: NodeJS.Timeout | null = null; +const changedFiles = new Set(); + +const watcher = await sandbox.watch('/workspace/src', { + onEvent: (event) => { + changedFiles.add(event.path); + + // Clear existing timeout + if (debounceTimeout) { + clearTimeout(debounceTimeout); + } + + // Set new timeout to process changes + debounceTimeout = setTimeout(async () => { + console.log(`Processing ${changedFiles.size} changed files...`); + + // Process all accumulated changes + for (const filePath of changedFiles) { + await processFile(filePath); + } + + changedFiles.clear(); + debounceTimeout = null; + }, 1000); // Wait 1 second after last change + } +}); + +async function processFile(filePath: string) { + // Your file processing logic + console.log(`Processing ${filePath}`); +} +``` + + +### Multi-directory watching + +Watch multiple directories with different configurations: + + +``` +// Watch source code for builds +const srcWatcher = await sandbox.watch('/workspace/src', { + include: ['*.ts', '*.tsx'], + onEvent: async (event) => { + if (event.type === 'modify') { + await sandbox.exec('npm run build:src'); + } + } +}); + +// Watch tests for test runs +const testWatcher = await sandbox.watch('/workspace/tests', { + include: ['*.test.ts', '*.spec.ts'], + onEvent: async (event) => { + if (event.type === 'modify') { + await sandbox.exec(`npm test -- ${event.path}`); + } + } +}); + +// Watch config files for full rebuilds +const configWatcher = await sandbox.watch('/workspace', { + include: ['package.json', 'tsconfig.json', 'vite.config.ts'], + recursive: false, // Only watch root level + onEvent: async (event) => { + console.log('Configuration changed, rebuilding project...'); + await sandbox.exec('npm run build'); + } +}); +``` + + +### Graceful shutdown + +Use AbortSignal for clean shutdown handling: + + +``` +const controller = new AbortController(); + +const watcher = await sandbox.watch('/workspace/src', { + signal: controller.signal, + onEvent: (event) => { + console.log(`Event: ${event.type} - ${event.path}`); + }, + onError: (error) => { + if (error.name === 'AbortError') { + console.log('Watch cancelled gracefully'); + } else { + console.error('Watch error:', error); + } + } +}); + +// Handle shutdown signals +process.on('SIGINT', () => { + console.log('Shutting down file watcher...'); + controller.abort(); +}); +``` + + +## Best practices + +### Resource management + +Always stop watchers to prevent resource leaks: + + +``` +const watchers: Array<{ stop: () => Promise }> = []; + +// Create watchers +const srcWatcher = await sandbox.watch('/workspace/src', options); +const testWatcher = await sandbox.watch('/workspace/tests', options); +watchers.push(srcWatcher, testWatcher); + +// Clean shutdown +async function shutdown() { + console.log('Stopping all watchers...'); + await Promise.all(watchers.map(w => w.stop())); + console.log('All watchers stopped'); +} + +process.on('exit', shutdown); +process.on('SIGINT', shutdown); +process.on('SIGTERM', shutdown); +``` + + +### Error handling + +Implement robust error handling for production use: + + +``` +const watcher = await sandbox.watch('/workspace/src', { + onEvent: async (event) => { + try { + await handleFileChange(event); + } catch (error) { + console.error(`Failed to handle ${event.type} event for ${event.path}:`, error); + // Don't let errors stop the watcher + } + }, + onError: async (error) => { + console.error('Watch system error:', error); + + // Attempt to restart watcher on critical errors + if (error.message.includes('inotify')) { + console.log('Attempting to restart file watcher...'); + await watcher.stop(); + // Recreate watcher with same options + } + } +}); +``` + + +### Performance optimization + +For high-frequency changes, use server-side filtering: + + +``` +// Efficient - filtering happens at kernel/inotify level +const watcher = await sandbox.watch('/workspace', { + include: ['*.ts'], // Only TypeScript files + exclude: ['node_modules'] // Skip dependencies +}); + +// Less efficient - all events sent to JavaScript +const watcher = await sandbox.watch('/workspace', { + onEvent: (event) => { + if (!event.path.endsWith('.ts')) return; + if (event.path.includes('node_modules')) return; + // Handle event + } +}); +``` + + +## Troubleshooting + +### Path not found errors + +Ensure directories exist before watching them: + + +``` +const watchPath = '/workspace/src'; + +// Check if path exists first +try { + const exists = await sandbox.readDir(watchPath); + const watcher = await sandbox.watch(watchPath, options); +} catch (error) { + if (error.message.includes('not found')) { + console.log(`Creating directory ${watchPath}...`); + await sandbox.exec(`mkdir -p ${watchPath}`); + // Now start watching + const watcher = await sandbox.watch(watchPath, options); + } +} +``` + + +### High CPU usage + +If watching large directories causes performance issues: + +1. Use specific `include` patterns instead of watching everything +2. Exclude large directories like `node_modules` and `dist` +3. Watch specific subdirectories instead of the entire project +4. Use non-recursive watching for shallow monitoring + +:::note[Container lifecycle] +File watchers are automatically stopped when the sandbox sleeps or shuts down. They will restart when the sandbox wakes up, but you may need to re-establish them in your application logic. +::: + +## Related resources + +- [File Watching API reference](/sandbox/api/file-watching/) - Complete API documentation +- [Manage files guide](/sandbox/guides/manage-files/) - File operations +- [Background processes guide](/sandbox/guides/background-processes/) - Long-running processes diff --git a/src/content/docs/sandbox/guides/watch-files.mdx b/src/content/docs/sandbox/guides/watch-files.mdx new file mode 100644 index 000000000000000..9efa70a2bb5e2cb --- /dev/null +++ b/src/content/docs/sandbox/guides/watch-files.mdx @@ -0,0 +1,425 @@ +--- +title: Watch for file changes +pcx_content_type: how-to +sidebar: + order: 8 +description: Monitor filesystem changes in real-time for hot reloading, automated testing, and live file synchronization. +--- + +import { TypeScriptExample } from "~/components"; + +This guide shows you how to monitor filesystem changes in real-time using the file watching API. You'll learn to set up watches, handle events, filter changes, and build common patterns like hot reloading. + +## Basic file watching + +Start by watching a directory for all changes: + + +``` +import { getSandbox } from '@cloudflare/sandbox'; + +const sandbox = getSandbox(env.Sandbox, 'my-sandbox'); + +const watcher = await sandbox.watch('/workspace/src', { + onEvent: (event) => { + console.log(`${event.type}: ${event.path}`); + + if (event.isDirectory) { + console.log('Directory changed'); + } else { + console.log('File changed'); + } + }, + onError: (error) => { + console.error('Watch failed:', error); + } +}); + +console.log(`Started watching: ${watcher.path}`); +``` + + +The watcher will immediately start monitoring the specified path and call your `onEvent` callback for each filesystem change. + +## Filter watched files + +Use include and exclude patterns to focus on relevant files: + + +``` +// Watch only JavaScript and TypeScript files +const jsWatcher = await sandbox.watch('/workspace', { + include: ['*.js', '*.ts', '*.jsx', '*.tsx'], + onEvent: (event) => { + console.log(`JS/TS file ${event.type}: ${event.path}`); + } +}); + +// Watch everything except logs and build artifacts +const filtered = await sandbox.watch('/workspace', { + exclude: ['*.log', 'dist/*', 'build/*', 'node_modules/*'], + onEvent: (event) => { + console.log(`Source change: ${event.path}`); + } +}); +``` + + +:::note[Pattern matching] +Glob patterns match against the filename only, not the full path. Use patterns like `*.ts` for TypeScript files or `test*` for files starting with "test". +::: + +## Hot reloading server + +Build a hot reloading development server that restarts when code changes: + + +``` +import { getSandbox } from '@cloudflare/sandbox'; + +async function setupHotReload(sandbox) { + let serverProcess = null; + + // Start the initial server + async function startServer() { + if (serverProcess) { + await serverProcess.kill(); + } + + console.log('Starting development server...'); + serverProcess = await sandbox.startProcess({ + command: 'npm run dev', + cwd: '/workspace' + }); + + console.log('Server started, waiting for changes...'); + } + + // Watch for source code changes + const watcher = await sandbox.watch('/workspace/src', { + include: ['*.js', '*.ts', '*.jsx', '*.tsx'], + onEvent: async (event) => { + if (event.type === 'modify' || event.type === 'create') { + console.log(`Code changed: ${event.path}`); + console.log('Restarting server...'); + await startServer(); + } + }, + onError: (error) => { + console.error('Watch error:', error); + } + }); + + // Start initial server + await startServer(); + + return { watcher, serverProcess }; +} + +// Usage +const sandbox = getSandbox(env.Sandbox, 'dev-sandbox'); +const { watcher } = await setupHotReload(sandbox); + +// Stop watching when done +// await watcher.stop(); +``` + + +## Test runner automation + +Automatically run tests when source files change: + + +``` +async function setupAutoTesting(sandbox) { + let isTestRunning = false; + + async function runTests() { + if (isTestRunning) { + console.log('Tests already running, skipping...'); + return; + } + + isTestRunning = true; + console.log('Running tests...'); + + try { + const result = await sandbox.exec('npm test', { + cwd: '/workspace' + }); + + if (result.exitCode === 0) { + console.log('✅ Tests passed'); + } else { + console.log('❌ Tests failed'); + console.log(result.stderr); + } + } catch (error) { + console.error('Test execution failed:', error); + } finally { + isTestRunning = false; + } + } + + const watcher = await sandbox.watch('/workspace', { + include: ['*.js', '*.ts', '*.test.js', '*.spec.ts'], + onEvent: async (event) => { + if (event.type === 'modify' || event.type === 'create') { + console.log(`File changed: ${event.path}`); + // Debounce rapid changes + setTimeout(runTests, 500); + } + } + }); + + return watcher; +} + +const sandbox = getSandbox(env.Sandbox, 'test-sandbox'); +await setupAutoTesting(sandbox); +``` + + +## File synchronization + +Sync files between local development and the sandbox: + + +``` +async function syncFiles(sandbox, localFiles) { + const watcher = await sandbox.watch('/workspace', { + onEvent: async (event) => { + // Sync sandbox changes back to local storage + if (event.type === 'modify' && !event.isDirectory) { + try { + const file = await sandbox.readFile(event.path); + await storeFile(event.path, file.content); + console.log(`Synced: ${event.path}`); + } catch (error) { + console.error(`Failed to sync ${event.path}:`, error); + } + } + } + }); + + // Upload local files to sandbox + for (const [path, content] of Object.entries(localFiles)) { + await sandbox.writeFile(`/workspace/${path}`, content); + } + + return watcher; +} + +// Mock storage functions +async function storeFile(path, content) { + // Store to your preferred backend (R2, KV, etc.) +} + +// Usage +const localFiles = { + 'app.js': 'console.log("Hello World");', + 'package.json': JSON.stringify({ name: 'my-app' }) +}; + +const sandbox = getSandbox(env.Sandbox, 'sync-sandbox'); +const watcher = await syncFiles(sandbox, localFiles); +``` + + +## Handle watch lifecycle + +Properly manage watch lifecycle with AbortController: + + +``` +async function managedWatch(sandbox, duration = 60000) { + const controller = new AbortController(); + + // Auto-cleanup after duration + const timeout = setTimeout(() => { + console.log('Auto-stopping watch...'); + controller.abort(); + }, duration); + + try { + const watcher = await sandbox.watch('/workspace', { + signal: controller.signal, + onEvent: (event) => { + console.log(`Change: ${event.path}`); + }, + onError: (error) => { + if (error.name === 'AbortError') { + console.log('Watch cancelled'); + } else { + console.error('Watch error:', error); + } + } + }); + + console.log('Watch started, will auto-stop in 60s'); + return watcher; + + } finally { + clearTimeout(timeout); + } +} + +// Usage +const sandbox = getSandbox(env.Sandbox, 'temp-sandbox'); +await managedWatch(sandbox, 30000); // 30 second watch +``` + + +## Performance optimization + +For large directories, optimize watch performance: + + +``` +// Efficient watching for large projects +const watcher = await sandbox.watch('/workspace', { + // Only watch source directories + recursive: true, + + // Include only relevant file types + include: [ + '*.js', '*.ts', '*.jsx', '*.tsx', // JavaScript/TypeScript + '*.json', '*.md', // Config and docs + '*.css', '*.scss' // Styles + ], + + // Exclude heavy directories (already in defaults) + exclude: [ + '.git', 'node_modules', '.DS_Store', + '*.log', 'dist', 'build', '.next', + 'coverage', '.nyc_output' + ], + + onEvent: (event) => { + // Process only meaningful events + if (event.type === 'modify' && !event.isDirectory) { + handleFileChange(event.path); + } + } +}); + +function handleFileChange(path) { + // Your change handling logic + console.log(`Processing: ${path}`); +} +``` + + +## Error recovery + +Handle watch failures gracefully: + + +``` +async function createResilientWatch(sandbox, path, options = {}) { + let attempts = 0; + const maxAttempts = 3; + + async function setupWatch() { + attempts++; + + try { + return await sandbox.watch(path, { + ...options, + onError: async (error) => { + console.error(`Watch error (attempt ${attempts}):`, error); + + if (attempts < maxAttempts) { + console.log('Attempting to restart watch...'); + setTimeout(setupWatch, 2000); // Wait 2s before retry + } else { + console.error('Max watch restart attempts reached'); + } + } + }); + } catch (error) { + if (attempts < maxAttempts) { + console.log(`Watch setup failed, retrying in 2s... (${attempts}/${maxAttempts})`); + setTimeout(setupWatch, 2000); + } else { + throw new Error(`Failed to setup watch after ${maxAttempts} attempts`); + } + } + } + + return setupWatch(); +} + +// Usage with retry logic +const sandbox = getSandbox(env.Sandbox, 'resilient-sandbox'); +const watcher = await createResilientWatch(sandbox, '/workspace', { + onEvent: (event) => console.log(event) +}); +``` + + +## Multiple watchers + +Coordinate multiple watches for different purposes: + + +``` +async function setupMultiWatch(sandbox) { + const watchers = []; + + // Watch source code for hot reload + const srcWatcher = await sandbox.watch('/workspace/src', { + include: ['*.js', '*.ts'], + onEvent: (event) => handleSourceChange(event) + }); + watchers.push(srcWatcher); + + // Watch config files for restart + const configWatcher = await sandbox.watch('/workspace', { + include: ['package.json', '*.config.js', '.env*'], + onEvent: (event) => handleConfigChange(event) + }); + watchers.push(configWatcher); + + // Watch tests for auto-run + const testWatcher = await sandbox.watch('/workspace/tests', { + include: ['*.test.js', '*.spec.ts'], + onEvent: (event) => handleTestChange(event) + }); + watchers.push(testWatcher); + + // Cleanup function + const stopAll = async () => { + await Promise.all(watchers.map(w => w.stop())); + console.log('All watchers stopped'); + }; + + return { watchers, stopAll }; +} + +function handleSourceChange(event) { + console.log(`Source changed: ${event.path}`); + // Hot reload logic +} + +function handleConfigChange(event) { + console.log(`Config changed: ${event.path}`); + // Restart server logic +} + +function handleTestChange(event) { + console.log(`Test changed: ${event.path}`); + // Run specific tests +} + +// Usage +const sandbox = getSandbox(env.Sandbox, 'multi-watch'); +const { stopAll } = await setupMultiWatch(sandbox); + +// Later: await stopAll(); +``` + + +:::note[Resource management] +Each watch consumes system resources. Stop watches when they are no longer needed using `await watcher.stop()` or by aborting with `AbortController`. +::: \ No newline at end of file diff --git a/src/content/docs/sandbox/index.mdx b/src/content/docs/sandbox/index.mdx index 434fa3641586487..5e97e6b004c685a 100644 --- a/src/content/docs/sandbox/index.mdx +++ b/src/content/docs/sandbox/index.mdx @@ -120,6 +120,40 @@ With Sandbox, you can execute Python scripts, run Node.js applications, analyze ``` + + ```typescript + import { getSandbox } from '@cloudflare/sandbox'; + + export { Sandbox } from '@cloudflare/sandbox'; + + export default { + async fetch(request: Request, env: Env): Promise { + const sandbox = getSandbox(env.Sandbox, 'user-123'); + + // Watch for file changes in real-time + const watcher = await sandbox.watch('/workspace/src', { + include: ['*.js', '*.ts'], + onEvent: (event) => { + console.log(`${event.type}: ${event.path}`); + if (event.type === 'modify') { + // Trigger rebuild or hot reload + console.log('Code changed, recompiling...'); + } + }, + onError: (error) => { + console.error('Watch error:', error); + } + }); + + // Stop watching when done + setTimeout(() => watcher.stop(), 60000); + + return Response.json({ message: 'File watcher started' }); + } + }; + ``` + + ```typescript import { getSandbox } from '@cloudflare/sandbox'; @@ -185,6 +219,12 @@ Mount S3-compatible object storage (R2, S3, GCS, and more) as local filesystems. + + +Monitor files and directories for changes using native filesystem events. Perfect for building hot reloading development servers, build automation systems, and configuration monitoring tools. + + + --- ## Use Cases diff --git a/src/content/partials/workers/wrangler-commands/r2.mdx b/src/content/partials/workers/wrangler-commands/r2.mdx index 626a935c84c27e6..3046ac2f72125bb 100644 --- a/src/content/partials/workers/wrangler-commands/r2.mdx +++ b/src/content/partials/workers/wrangler-commands/r2.mdx @@ -50,6 +50,9 @@ npx wrangler r2 bucket catalog compaction disable my-bucket # Disable table-level compaction npx wrangler r2 bucket catalog compaction disable my-bucket my-namespace my-table ``` + + +