Files for Claude to work with, for the duration of the flags migrationproject

Signed-off-by: Rohit Nayak <[email protected]>

Author: rohit-nayak-ps

go/cmd/vtbackup/cli/vtbackup.go

@@ -201,25 +201,25 @@ func init() {
 
 	servenv.MoveFlagsToCobraCommand(Main)
 
-	Main.Flags().DurationVar(&minBackupInterval, "min_backup_interval", minBackupInterval, "Only take a new backup if it's been at least this long since the most recent backup.")
-	Main.Flags().DurationVar(&minRetentionTime, "min_retention_time", minRetentionTime, "Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups.")
-	Main.Flags().IntVar(&minRetentionCount, "min_retention_count", minRetentionCount, "Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made")
-	Main.Flags().BoolVar(&initialBackup, "initial_backup", initialBackup, "Instead of restoring from backup, initialize an empty database with the provided init-db-sql-file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed).")
-	Main.Flags().BoolVar(&allowFirstBackup, "allow_first_backup", allowFirstBackup, "Allow this job to take the first backup of an existing shard.")
-	Main.Flags().BoolVar(&restartBeforeBackup, "restart_before_backup", restartBeforeBackup, "Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs.")
+	utils.SetFlagDurationVar(Main.Flags(), &minBackupInterval, "min-backup-interval", minBackupInterval, "Only take a new backup if it's been at least this long since the most recent backup.")
+	utils.SetFlagDurationVar(Main.Flags(), &minRetentionTime, "min-retention-time", minRetentionTime, "Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups.")
+	utils.SetFlagIntVar(Main.Flags(), &minRetentionCount, "min-retention-count", minRetentionCount, "Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made")
+	utils.SetFlagBoolVar(Main.Flags(), &initialBackup, "initial-backup", initialBackup, "Instead of restoring from backup, initialize an empty database with the provided init-db-sql-file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed).")
+	utils.SetFlagBoolVar(Main.Flags(), &allowFirstBackup, "allow-first-backup", allowFirstBackup, "Allow this job to take the first backup of an existing shard.")
+	utils.SetFlagBoolVar(Main.Flags(), &restartBeforeBackup, "restart-before-backup", restartBeforeBackup, "Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs.")
 	Main.Flags().BoolVar(&upgradeSafe, "upgrade-safe", upgradeSafe, "Whether to use innodb_fast_shutdown=0 for the backup so it is safe to use for MySQL upgrades.")
 
 	// vttablet-like flags
 	utils.SetFlagStringVar(Main.Flags(), &initDbNameOverride, "init-db-name-override", initDbNameOverride, "(init parameter) override the name of the db used by vttablet")
 	utils.SetFlagStringVar(Main.Flags(), &initKeyspace, "init-keyspace", initKeyspace, "(init parameter) keyspace to use for this tablet")
 	utils.SetFlagStringVar(Main.Flags(), &initShard, "init-shard", initShard, "(init parameter) shard to use for this tablet")
 	Main.Flags().IntVar(&concurrency, "concurrency", concurrency, "(init restore parameter) how many concurrent files to restore at once")
-	Main.Flags().StringVar(&incrementalFromPos, "incremental_from_pos", incrementalFromPos, "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
+	utils.SetFlagStringVar(Main.Flags(), &incrementalFromPos, "incremental-from-pos", incrementalFromPos, "Position, or name of backup from which to create an incremental backup. Default: empty. If given, then this backup becomes an incremental backup from given position or given backup. If value is 'auto', this backup will be taken from the last successful backup position.")
 
 	// mysqlctld-like flags
 	utils.SetFlagIntVar(Main.Flags(), &mysqlPort, "mysql-port", mysqlPort, "MySQL port")
 	utils.SetFlagStringVar(Main.Flags(), &mysqlSocket, "mysql-socket", mysqlSocket, "Path to the mysqld socket file")
-	Main.Flags().DurationVar(&mysqlTimeout, "mysql_timeout", mysqlTimeout, "how long to wait for mysqld startup")
+	utils.SetFlagDurationVar(Main.Flags(), &mysqlTimeout, "mysql-timeout", mysqlTimeout, "how long to wait for mysqld startup")
 	Main.Flags().DurationVar(&mysqlShutdownTimeout, "mysql-shutdown-timeout", mysqlShutdownTimeout, "how long to wait for mysqld shutdown")
 	utils.SetFlagStringVar(Main.Flags(), &initDBSQLFile, "init-db-sql-file", initDBSQLFile, "path to .sql file to run after mysql_install_db")
 	Main.Flags().BoolVar(&detachedMode, "detach", detachedMode, "detached mode - run backups detached from the terminal")

go/cmd/vtctlclient/main.go

@@ -28,11 +28,11 @@ import (
 
 	"vitess.io/vitess/go/acl"
 	"vitess.io/vitess/go/exit"
-	"vitess.io/vitess/go/vt/utils"
 	"vitess.io/vitess/go/trace"
 	"vitess.io/vitess/go/vt/log"
 	"vitess.io/vitess/go/vt/logutil"
 	"vitess.io/vitess/go/vt/servenv"
+	"vitess.io/vitess/go/vt/utils"
 	"vitess.io/vitess/go/vt/vtctl/vtctlclient"
 
 	logutilpb "vitess.io/vitess/go/vt/proto/logutil"

go/flags/endtoend/FLAGS_PROJECT_README.md

@@ -16,10 +16,33 @@ You can find some PRs that have done this in the list
 from: https://github.com/vitessio/vitess/pulls?q=is%3Apr+flags+author%3Amounicasruthi+is%3Amerged
 For example: https://github.com/vitessio/vitess/pull/17975, https://github.com/vitessio/vitess/pull/18296
 
+
+## Pending Tasks
+
+1. Find all the flags per binary by writing a similar script like count_flags.sh that 
+    * per binary, lists all flags for tha binary
+    * per flag, lists all binaries for that flag
+    * count of all binaries pending with number of flags pending for each
+2. Group flags by function 
+3. Replace the flags across the code base using the flags utility functions from go/vt/utils/flags.go as done in 
+   previous PRs
+4. Ensure unit tests run in the folders where the files are being modified
+5. Pause after each group for manual validation and check-in and potentially separate PR creation
+6. Repeat Steps 3 to 5 until all flags are replaced
+7. Finally, go through the code base looking for any flags with underscores still being used and see how we can 
+   replace that with the dashed version. For this, since there are several identifiers using underscores, we should
+   do a grep for the underscore versions of the all the flags either one by one or with a giant regexp, which ever 
+   works best. Or we could also create a golang program that goes through every go file in the repository. Whichever 
+   is the best approach, we are ok running these programs for long durations, say overnight, if required on our 
+   local machines.
+
 ## Miscellaneous notes
 
 * We don't want to extend the usage of DualFormatVar for flags, because technically that supports a mix of
   dashes and underscores, and that is not what we want to do.
+* Sometimes tests fail due to upgrade/downgrade and we need to use the `GetFlagVariantForTestsByVersion` function 
+  but that needs to be manually determined from the failures in the CI upgrade/downgrade tests. If you see this 
+  function being used DO NOT replace it.
 
 * The binaries we are interested in:
     * vtctld

go/flags/endtoend/MIGRATION_CHECKLIST.md

@@ -0,0 +1,88 @@
+# Vitess Flags Migration Verification Checklist
+
+## Pre-Migration Checklist
+- [x] Analyze current flag usage with help text files
+- [x] Group flags by function/component
+- [x] Identify high-impact flags (used by multiple binaries)
+- [x] Plan migration order
+
+## Per-Group Migration Checklist
+
+### For Each Flag Group:
+1. **Code Changes**
+   - [ ] Add `vitess.io/vitess/go/vt/utils` import where needed
+   - [ ] Replace flag definition with appropriate `utils.SetFlag*Var` function
+   - [ ] Ensure flag name uses dashes, not underscores
+   - [ ] DO NOT manually add deprecation/hiding - utils functions handle this automatically
+   - [ ] For cobra-based commands, use utils functions with Root.PersistentFlags() or cmd.Flags()
+
+2. **Testing**
+   - [ ] Run unit tests for modified packages
+   - [ ] Verify both flag formats work (--flag-name and --flag_name)
+   - [ ] Check that underscore version shows deprecation warning
+   - [ ] Ensure help text shows dashed version as primary
+
+3. **Documentation**
+   - [ ] Update CLAUDE.md with completed migrations
+   - [ ] Note any special cases or exceptions found
+   - [ ] Record test results
+
+## Completed Migrations
+
+### ✅ Timeout Flags Group (2 flags)
+**Date:** 2025-09-05
+**Files Modified:**
+- `/go/vt/vtctld/action_repository.go`
+- `/go/cmd/vtctlclient/main.go`
+- `/go/cmd/vtctldclient/command/root.go`
+- `/go/vt/vtgate/tabletgateway.go`
+
+**Flags Migrated:**
+- `action_timeout` → `action-timeout`
+- `gateway_initial_tablet_timeout` → `gateway-initial-tablet-timeout`
+
+**Tests Run:**
+- ✅ `go test ./go/vt/vtctld` - PASS
+- ✅ `go test ./go/vt/vtgate` - PASS
+- ✅ Help text verification - Shows dashed version
+
+## Pending Migrations
+
+### Backup/Restore Flags (5 flags)
+- `allow_first_backup`
+- `file_backup_storage_root`
+- `initial_backup`
+- `min_backup_interval`
+- `restart_before_backup`
+
+### Database Flags (20 flags)
+- Various mysql_auth_* flags
+- mysql_bind_host
+- mysql_timeout
+- etc.
+
+### Stats/Monitoring Flags (2 flags)
+- `statsd_address`
+- `statsd_sample_rate`
+
+## Final Verification
+- [ ] All underscore flags replaced with dash versions
+- [ ] Backward compatibility maintained
+- [ ] All tests passing
+- [ ] Documentation updated
+- [ ] PR created and reviewed
+
+## Notes
+- Glog flags (log_dir, log_backtrace_at) come from Go standard library - may not need migration
+- Some tests use `GetFlagVariantForTestsByVersion` - DO NOT modify these
+- Utils functions automatically handle deprecation and hiding - no manual work needed
+- Utils functions work with both pflag.FlagSet and cobra's Flags()/PersistentFlags()
+
+## Key Pattern to Follow:
+```go
+// Before:
+fs.StringVar(&variable, "flag_with_underscore", defaultValue, "description")
+
+// After:
+utils.SetFlagStringVar(fs, &variable, "flag-with-dashes", defaultValue, "description")
+```

go/flags/endtoend/analyze_flags.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Comprehensive flag analysis tool for Vitess flags migration project.
+Analyzes help text files to identify flags with underscores that need migration.
+"""
+
+import os
+import re
+from collections import defaultdict
+import json
+
+def parse_help_file(filepath):
+    """Parse a help text file and extract flag information."""
+    flags = []
+    binary_name = os.path.basename(filepath).replace('.txt', '')
+    
+    with open(filepath, 'r') as f:
+        lines = f.readlines()
+    
+    reading_flags = False
+    for line in lines:
+        # Check if we're entering the flags section
+        if line.strip() in ["Flags:", "Usage of vtctlclient:"]:
+            reading_flags = True
+            continue
+        
+        if reading_flags:
+            # Extract flag if line starts with --
+            match = re.match(r'\s*(--[a-zA-Z0-9_-]+)', line)
+            if match:
+                flag_name = match.group(1)[2:]  # Remove --
+                has_underscore = '_' in flag_name
+                has_dash = '-' in flag_name
+                
+                flags.append({
+                    'name': flag_name,
+                    'has_underscore': has_underscore,
+                    'has_dash': has_dash,
+                    'binary': binary_name
+                })
+    
+    return flags
+
+def analyze_all_binaries():
+    """Analyze all binary help files and generate comprehensive report."""
+    flags_dir = '/Users/rohit/vitess/go/flags/endtoend'
+    
+    # Target binaries we care about
+    target_binaries = [
+        'vtctld', 'vtctldclient', 'vttablet', 'vtgate', 'vtorc',
+        'mysqlctl', 'mysqlctld', 'vtcombo', 'vttestserver', 
+        'vtbackup', 'vtclient', 'vtbench'
+    ]
+    
+    all_flags = []
+    binary_stats = {}
+    flag_to_binaries = defaultdict(set)
+    
+    # Process each help file
+    for filename in os.listdir(flags_dir):
+        if filename.endswith('.txt'):
+            binary_name = filename.replace('.txt', '')
+            if binary_name not in target_binaries:
+                continue
+                
+            filepath = os.path.join(flags_dir, filename)
+            flags = parse_help_file(filepath)
+            all_flags.extend(flags)
+            
+            # Calculate stats for this binary
+            underscore_count = sum(1 for f in flags if f['has_underscore'])
+            dash_count = sum(1 for f in flags if f['has_dash'] and not f['has_underscore'])
+            
+            binary_stats[binary_name] = {
+                'total_flags': len(flags),
+                'underscore_flags': underscore_count,
+                'dash_flags': dash_count,
+                'completion_percentage': round((dash_count / len(flags) * 100) if flags else 100, 1)
+            }
+            
+            # Map flags to binaries
+            for flag in flags:
+                flag_to_binaries[flag['name']].add(binary_name)
+    
+    return all_flags, binary_stats, flag_to_binaries
+
+def generate_report():
+    """Generate comprehensive migration report."""
+    all_flags, binary_stats, flag_to_binaries = analyze_all_binaries()
+    
+    print("=" * 80)
+    print("VITESS FLAGS MIGRATION STATUS REPORT")
+    print("=" * 80)
+    print()
+    
+    # Overall statistics
+    total_underscore = sum(s['underscore_flags'] for s in binary_stats.values())
+    total_dash = sum(s['dash_flags'] for s in binary_stats.values())
+    total_flags = sum(s['total_flags'] for s in binary_stats.values())
+    
+    print("OVERALL PROGRESS:")
+    print(f"  Total flags: {total_flags}")
+    print(f"  Flags with underscores (need migration): {total_underscore}")
+    print(f"  Flags with dashes (completed): {total_dash}")
+    print(f"  Overall completion: {round((total_dash / total_flags * 100) if total_flags else 100, 1)}%")
+    print()
+    
+    # Per-binary statistics
+    print("PER-BINARY STATUS:")
+    print("-" * 80)
+    print(f"{'Binary':<15} {'Total':<10} {'Underscore':<12} {'Dash':<10} {'Completion':<12}")
+    print("-" * 80)
+    
+    # Sort by number of underscore flags (most work needed first)
+    sorted_binaries = sorted(binary_stats.items(), 
+                            key=lambda x: x[1]['underscore_flags'], 
+                            reverse=True)
+    
+    for binary, stats in sorted_binaries:
+        status = "✓ DONE" if stats['underscore_flags'] == 0 else ""
+        print(f"{binary:<15} {stats['total_flags']:<10} {stats['underscore_flags']:<12} "
+              f"{stats['dash_flags']:<10} {stats['completion_percentage']:<10.1f}% {status}")
+    
+    print()
+    
+    # Find unique underscore flags across all binaries
+    unique_underscore_flags = set()
+    for flag in all_flags:
+        if flag['has_underscore']:
+            unique_underscore_flags.add(flag['name'])
+    
+    print(f"UNIQUE FLAGS WITH UNDERSCORES: {len(unique_underscore_flags)}")
+    print("-" * 80)
+    
+    # Group flags by how many binaries use them
+    flag_usage = defaultdict(list)
+    for flag_name in unique_underscore_flags:
+        binaries = flag_to_binaries[flag_name]
+        flag_usage[len(binaries)].append((flag_name, binaries))
+    
+    # Show flags used by multiple binaries first (higher impact)
+    for count in sorted(flag_usage.keys(), reverse=True):
+        if count > 1:
+            print(f"\nFlags used by {count} binaries:")
+            for flag_name, binaries in sorted(flag_usage[count]):
+                print(f"  {flag_name:<40} -> {', '.join(sorted(binaries))}")
+    
+    # Save detailed data to JSON for further processing
+    detailed_data = {
+        'binary_stats': binary_stats,
+        'unique_underscore_flags': list(unique_underscore_flags),
+        'flag_to_binaries': {k: list(v) for k, v in flag_to_binaries.items() if '_' in k}
+    }
+    
+    with open('/Users/rohit/vitess/go/flags/endtoend/flags_analysis.json', 'w') as f:
+        json.dump(detailed_data, f, indent=2)
+    
+    print()
+    print("Detailed analysis saved to: flags_analysis.json")
+    print()
+    
+    # Suggest migration priority
+    print("SUGGESTED MIGRATION PRIORITY:")
+    print("-" * 80)
+    print("1. High-impact flags (used by multiple binaries)")
+    print("2. Binaries with most underscore flags remaining:")
+    for i, (binary, stats) in enumerate(sorted_binaries[:5], 1):
+        if stats['underscore_flags'] > 0:
+            print(f"   {i}. {binary} ({stats['underscore_flags']} flags)")
+
+if __name__ == "__main__":
+    generate_report()