Various correctness, robustness, and performance fixes

Validation and routing:
- Add word-boundary checks to command validation so DESCRIBING/SELECTS/
  INSERTION etc. no longer match DESCRIBE/SELECT/INSERT prefixes
- Rewrite stripComments in router and batch/utils to respect quoted
  strings (was corrupting URLs and SQL with -- inside quotes)
- Use db.ConvertToJSONQuery for SELECT->SELECT JSON transform so
  SELECT DISTINCT is handled correctly

UDT decoder:
- Treat empty wire data for text/varchar/ascii as empty string and
  empty blob as []byte{}, instead of nil. Cassandra signals null with
  a -1 length prefix, not zero-length payload
- Drop defensive overflow checks on smallint/int/bigint that are
  impossible by two's-complement decoding; annotate with nolint:gosec
- Fix DATE encoding test: Cassandra DATE is days_since_epoch + 2^31,
  not raw int32. Add pre-epoch test case

Config robustness:
- Add OutputFormat field so default format can be set in config
- Surface cqlshrc permission/parse errors via LoadWarnings instead of
  silent swallowing
- Resolve relative credentials = path against the cqlshrc dir, not CWD
- Mask password/api/secret keys in debug logs; stop logging raw
  credentials-file lines

Session manager:
- SetKeyspace and SetOutputFormat now return error
- Validate keyspace name against ^[a-zA-Z_][a-zA-Z0-9_]*$
- Read default OutputFormat from config

DB session/executor:
- Read pageSize from config (was hardcoded 100)
- Remove bogus close-iterator-then-recreate dance from
  ExecuteSelectQuery that always ran the query twice

Batch:
- Quote-aware splitter and comment stripper (handles ''-escapes and
  BATCH boundaries)
- Precompute column-name index map in CSV output (O(rows*cols)
  instead of O(rows*cols^2))

AI:
- Validate extracted JSON with json.Valid; add extractBalancedJSON to
  match nested objects instead of LastIndex("}")
- Bound conversation cache to 100 entries with 24h TTL eviction
- Debounce search-index "last accessed" updates (every 30s, batched)
  instead of one goroutine per match
- Switch ai_handler to NewAIWithCache

Parquet/COPY:
- Sort partition column names so output is deterministic across runs
- Track partition file size from disk after Flush for accurate
  maxFileSize accounting
- Fix COPY header-row leak: skip Data[0] which contains headers
- Narrow UUID heuristic to "name contains uuid" only; "id" was a
  false positive for columns like provider_id, valid

UI:
- sliding_window MaxRows=0 now means unlimited (memory limit only)
- Log SetKeyspace error after the new error return

Meta-commands:
- PAGING OFF now sets pageSize to 0 (server defaults) instead of
  10000, in both code paths

Add AGENTS.md repository contributor guide
Add tests for batch/utils, db/session, and validation/command

Author: hshimizu

AGENTS.md

@@ -0,0 +1,39 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+- `cmd/cqlai/main.go` is the CLI entry point for the `cqlai` binary.
+- `internal/` holds core application packages (parsing, routing, UI, storage, etc.).
+- `assets/` contains branding and UI assets used by docs and release artifacts.
+- `docs/` includes user guides and feature documentation (Parquet, AI config, batch mode).
+- `test/` hosts test suites; `test/parquet/` for Parquet I/O unit tests and `test/integration/` for Cassandra-backed integration tests.
+- `scripts/` and `bin/` hold helper scripts and build outputs.
+- `cqlai.json.example` shows configuration defaults.
+
+## Build, Test, and Development Commands
+- `make build`: build the binary into `bin/`.
+- `make run`: build and launch locally.
+- `make test`: run all Go tests with race detection and coverage output.
+- `make test-coverage`: generate `coverage.html` from `coverage.out`.
+- `make lint`: run `golangci-lint` if available, otherwise `go vet`.
+- `make fmt`: format with `go fmt` and `goimports` (if installed).
+- `make grammar`: regenerate ANTLR grammar sources in `internal/parser/grammar/`.
+
+## Coding Style & Naming Conventions
+- Go standard formatting is required; use `make fmt` before pushing.
+- Follow idiomatic Go naming (mixedCaps for exported identifiers, lowerCamel for locals).
+- Keep package names short and lowercase; avoid underscores unless required by tools.
+
+## Testing Guidelines
+- Unit tests use Go’s `testing` package and live alongside code as `*_test.go`.
+- Parquet unit tests: `go test -v ./test/parquet/...` or `./test/parquet/run_tests.sh`.
+- Integration tests require a running Cassandra instance; see `test/integration/README.md`.
+- No explicit coverage threshold is documented; aim to cover new logic.
+
+## Commit & Pull Request Guidelines
+- Commit messages follow a conventional style seen in history (`feat:`, `fix:`, `perf:`, `chore:`).
+- PRs should include a clear description, linked issues when applicable, and test notes.
+- For UI or output changes, add terminal screenshots or recordings when it improves review clarity.
+
+## Configuration & Security
+- Local config is read from `cqlai.json` or `~/.cqlai.json`.
+- Do not commit API keys or secrets; use `cqlai.json.example` as a template.

internal/ai/client.go

@@ -168,24 +168,73 @@ type Message struct {
 
 // extractJSON attempts to extract JSON from a text response
 func extractJSON(text string) string {
-	// Look for JSON between ```json and ``` markers
+	// First priority: Look for JSON between ```json and ``` markers
 	startMarker := JSONStartMarker
 	endMarker := JSONEndMarker
 	startIdx := strings.Index(text, startMarker)
 	if startIdx != -1 {
 		startIdx += len(startMarker)
 		endIdx := strings.Index(text[startIdx:], endMarker)
 		if endIdx != -1 {
-			return strings.TrimSpace(text[startIdx : startIdx+endIdx])
+			candidate := strings.TrimSpace(text[startIdx : startIdx+endIdx])
+			// Validate it's actual JSON
+			if json.Valid([]byte(candidate)) {
+				return candidate
+			}
 		}
 	}
 
-	// Look for JSON between { and }
+	// Second priority: Extract balanced JSON object
 	startIdx = strings.Index(text, "{")
 	if startIdx != -1 {
-		endIdx := strings.LastIndex(text, "}")
-		if endIdx != -1 && endIdx > startIdx {
-			return text[startIdx : endIdx+1]
+		// Find the matching closing brace by counting braces
+		candidate := extractBalancedJSON(text[startIdx:])
+		if candidate != "" && json.Valid([]byte(candidate)) {
+			return candidate
+		}
+	}
+
+	return ""
+}
+
+// extractBalancedJSON extracts a balanced JSON object from text starting with {
+func extractBalancedJSON(text string) string {
+	if len(text) == 0 || text[0] != '{' {
+		return ""
+	}
+
+	depth := 0
+	inString := false
+	escaped := false
+
+	for i, c := range text {
+		if escaped {
+			escaped = false
+			continue
+		}
+
+		if c == '\\' && inString {
+			escaped = true
+			continue
+		}
+
+		if c == '"' {
+			inString = !inString
+			continue
+		}
+
+		if inString {
+			continue
+		}
+
+		switch c {
+		case '{':
+			depth++
+		case '}':
+			depth--
+			if depth == 0 {
+				return text[:i+1]
+			}
 		}
 	}
 

internal/ai/conversation_manager.go

@@ -14,9 +14,11 @@ import (
 )
 
 const (
-	openAiBaseURL     = "https://api.openai.com/v1"
-	openRouterBaseURL = "https://openrouter.ai/api/v1"
-	ollamaBaseURL     = "http://localhost:11434/v1"
+	openAiBaseURL        = "https://api.openai.com/v1"
+	openRouterBaseURL    = "https://openrouter.ai/api/v1"
+	ollamaBaseURL        = "http://localhost:11434/v1"
+	maxConversations     = 100              // Maximum number of conversations to keep
+	conversationMaxAge   = 24 * time.Hour   // Maximum age before cleanup
 )
 
 // ConversationManager manages ongoing AI conversations
@@ -39,6 +41,11 @@ func (cm *ConversationManager) StartConversation(provider, model, apiKey, baseUR
 	cm.mu.Lock()
 	defer cm.mu.Unlock()
 
+	// Cleanup old conversations if we have too many
+	if len(cm.conversations) >= maxConversations {
+		cm.cleanupLocked()
+	}
+
 	conv := &AIConversation{
 		ID:              fmt.Sprintf("conv-%d", time.Now().UnixNano()),
 		Provider:        provider,
@@ -142,6 +149,48 @@ func (cm *ConversationManager) CleanupOldConversations(maxAge time.Duration) {
 	}
 }
 
+// cleanupLocked removes old conversations (must be called with lock held)
+func (cm *ConversationManager) cleanupLocked() {
+	cutoff := time.Now().Add(-conversationMaxAge)
+	cleaned := 0
+
+	// First pass: remove conversations older than maxAge
+	for id, conv := range cm.conversations {
+		if conv.LastActivity.Before(cutoff) {
+			delete(cm.conversations, id)
+			cleaned++
+			logger.DebugfToFile("ConversationManager", "Cleaned up old conversation %s", id)
+		}
+	}
+
+	// Second pass: if still over limit, remove oldest conversations
+	for len(cm.conversations) >= maxConversations {
+		var oldestID string
+		var oldestTime time.Time
+		first := true
+
+		for id, conv := range cm.conversations {
+			if first || conv.LastActivity.Before(oldestTime) {
+				oldestID = id
+				oldestTime = conv.LastActivity
+				first = false
+			}
+		}
+
+		if oldestID != "" {
+			delete(cm.conversations, oldestID)
+			cleaned++
+			logger.DebugfToFile("ConversationManager", "Evicted oldest conversation %s to stay under limit", oldestID)
+		} else {
+			break
+		}
+	}
+
+	if cleaned > 0 {
+		logger.DebugfToFile("ConversationManager", "Cleanup complete: removed %d conversations, %d remaining", cleaned, len(cm.conversations))
+	}
+}
+
 // Continue continues the conversation with user input (or empty string for continuation)
 func (conv *AIConversation) Continue(ctx context.Context, userInput string) (*AIResult, *InteractionRequest, error) {
 	conv.LastActivity = time.Now()

internal/ai/search_index_manager.go

@@ -21,12 +21,14 @@ type FuzzyMatch struct {
 // SearchIndexManager manages the search index for fuzzy matching
 // This is separate from schema cache to optimize performance
 type SearchIndexManager struct {
-	cache           *db.SchemaCache
-	tableIndex      map[string]*TableSearchEntry // keyspace.table -> search entry
-	lastIndexBuild  time.Time
-	indexTTL        time.Duration
-	mu              sync.RWMutex
-	buildInProgress bool
+	cache              *db.SchemaCache
+	tableIndex         map[string]*TableSearchEntry // keyspace.table -> search entry
+	lastIndexBuild     time.Time
+	lastAccessUpdate   time.Time // Last time we updated access timestamps (for debouncing)
+	indexTTL           time.Duration
+	mu                 sync.RWMutex
+	buildInProgress    bool
+	pendingAccessBatch []FuzzyMatch // Batched access updates
 }
 
 // TableSearchEntry contains search metadata for a table
@@ -132,14 +134,13 @@ func (sim *SearchIndexManager) FindTables(query string, limit int) []FuzzyMatch
 		logger.DebugfToFile("SearchIndexManager", "Failed to build index: %v", err)
 	}
 
-	sim.mu.RLock()
-	defer sim.mu.RUnlock()
-
 	queryLower := strings.ToLower(query)
 	queryTokens := tokenizeTableName(query)
 
 	var matches []FuzzyMatch
 
+	// Hold read lock only during the search
+	sim.mu.RLock()
 	for _, entry := range sim.tableIndex {
 		score := calculateFuzzyScore(queryLower, queryTokens, entry)
 
@@ -152,6 +153,7 @@ func (sim *SearchIndexManager) FindTables(query string, limit int) []FuzzyMatch
 			})
 		}
 	}
+	sim.mu.RUnlock()
 
 	// Sort by score (highest first)
 	sortFuzzyMatches(matches)
@@ -161,24 +163,39 @@ func (sim *SearchIndexManager) FindTables(query string, limit int) []FuzzyMatch
 		matches = matches[:limit]
 	}
 
-	// Update last accessed time for matched entries
-	go sim.updateLastAccessed(matches)
+	// Debounce last accessed time updates (only update every 30 seconds)
+	sim.maybeUpdateLastAccessed(matches)
 
 	return matches
 }
 
-// updateLastAccessed updates the last accessed time for matched entries
-func (sim *SearchIndexManager) updateLastAccessed(matches []FuzzyMatch) {
+// accessUpdateDebounce is the minimum interval between access timestamp updates
+const accessUpdateDebounce = 30 * time.Second
+
+// maybeUpdateLastAccessed updates access timestamps with debouncing
+// to avoid spawning goroutines for every FindTables call
+func (sim *SearchIndexManager) maybeUpdateLastAccessed(matches []FuzzyMatch) {
 	sim.mu.Lock()
 	defer sim.mu.Unlock()
 
+	// Batch the matches
+	sim.pendingAccessBatch = append(sim.pendingAccessBatch, matches...)
+
+	// Check if we should flush the batch
+	if time.Since(sim.lastAccessUpdate) < accessUpdateDebounce {
+		return
+	}
+
+	// Flush the batch
 	now := time.Now()
-	for _, match := range matches {
+	for _, match := range sim.pendingAccessBatch {
 		key := fmt.Sprintf("%s.%s", match.Keyspace, match.Table)
 		if entry, ok := sim.tableIndex[key]; ok {
 			entry.LastAccessed = now
 		}
 	}
+	sim.pendingAccessBatch = nil
+	sim.lastAccessUpdate = now
 }
 
 // calculateFuzzyScore calculates the fuzzy match score

internal/batch/executor.go

@@ -134,7 +134,8 @@ func NewExecutor(options *Options, writer io.Writer) (*Executor, error) {
 	// Create session manager for tracking keyspace changes
 	sessionMgr := session.NewManager(cfg)
 	if cfg.Keyspace != "" {
-		sessionMgr.SetKeyspace(cfg.Keyspace)
+		// SetKeyspace validates the keyspace name, but config values should already be valid
+		_ = sessionMgr.SetKeyspace(cfg.Keyspace)
 	}
 
 	// Initialize router with session manager
@@ -221,9 +222,9 @@ func (e *Executor) Execute(cql string) error {
 			keyspaceName := strings.TrimPrefix(v, "Now using keyspace ")
 			keyspaceName = strings.TrimSpace(keyspaceName)
 
-			// Update the session manager
+			// Update the session manager (keyspace already validated by Cassandra)
 			if e.sessionManager != nil {
-				e.sessionManager.SetKeyspace(keyspaceName)
+				_ = e.sessionManager.SetKeyspace(keyspaceName)
 			}
 
 			// Update the database session's keyspace