bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 199: large file handling

Browse Source
This commit is contained in:
Bill
2026-02-10 05:15:09 -07:00
parent ae1f8ba489
commit 02063766a8
10 changed files with 238 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
PROGRESS.md
View File

@@ -546,4 +546,5 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step
| 2026-02-10 | Codex | Step 196: High contrast + colorblind modes (high contrast theme, annotation shapes toggle, diagnostic pattern underlines). 1/1 tests pass (step196_test). |
| 2026-02-10 | Codex | Step 197: Keyboard navigation audit (F6 panel cycle, Esc focus return, focus ring defaults, nav focus enabled on panels). 1/1 tests pass (step197_test). |
| 2026-02-10 | Codex | Step 198: Virtual scrolling for large files (viewport + buffered rendering, fold hiding optimized). 1/1 tests pass (step198_test). |
| 2026-02-10 | Codex | Step 199: Large file handling (size thresholds, text-mode fallback, large file prompt, highlight disable, memory indicator). 1/1 tests pass (step199_test). |
| 2026-02-10 | Codex | Step 166: Extract main.cpp into panel headers marked complete (already implemented). step166_test passes; file_limits_test 4/4 passes. |

2
editor/CMakeLists.txt
View File

@@ -1134,6 +1134,8 @@ target_include_directories(step197_test PRIVATE src)
add_executable(step198_test tests/step198_test.cpp)
target_include_directories(step198_test PRIVATE src)
add_executable(step199_test tests/step199_test.cpp)
target_include_directories(step199_test PRIVATE src)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

79
editor/src/EditorState.h
View File

@@ -155,6 +155,9 @@ struct BufferState {
Orchestrator orchestrator;
bool orchestratorDirty = true;
int undoDepth = 0;
size_t fileSizeBytes = 0;
bool largeFileMode = false;
bool disableSyntaxHighlight = false;
};
struct EditorState {
@@ -178,6 +181,9 @@ struct EditorState {
EmacsState emacsState;
UIFlags ui;
UIAnimationState uiAnimations;
bool showLargeFilePrompt = false;
std::string largeFilePromptPath;
size_t largeFilePromptBytes = 0;
NotificationSystem notifications;
UIEventBus events;
@@ -583,7 +589,10 @@ struct EditorState {
void createBuffer(const std::string& path, const std::string& content,
const std::string& language,
BufferManager::BufferMode mode = BufferManager::BufferMode::Structured) {
BufferManager::BufferMode mode = BufferManager::BufferMode::Structured,
size_t fileSizeBytes = 0,
bool largeFileMode = false,
bool disableSyntaxHighlight = false) {
BufferManager::BufferMode effectiveMode = mode;
if (language == "org") effectiveMode = BufferManager::BufferMode::Text;
if (buffers.hasBuffer(path)) {
@@ -595,12 +604,17 @@ struct EditorState {
state->path = path;
state->language = language;
state->generatedLanguage = language;
state->fileSizeBytes = fileSizeBytes;
state->largeFileMode = largeFileMode;
state->disableSyntaxHighlight = disableSyntaxHighlight;
state->mode.setLanguage(language);
state->generatedMode.setLanguage(language);
state->editor.setContent(content, language);
state->sync.setText(content, language);
state->sync.syncNow();
state->incrementalOptimizer.setRoot(state->sync.getAST());
if (effectiveMode == BufferManager::BufferMode::Structured) {
state->sync.setText(content, language);
state->sync.syncNow();
state->incrementalOptimizer.setRoot(state->sync.getAST());
}
state->editBuf = content;
state->highlightsDirty = true;
state->generatedHighlightsDirty = true;
@@ -932,6 +946,20 @@ struct EditorState {
}
}
void setActiveBufferMode(BufferManager::BufferMode mode) {
if (!active()) return;
if (active()->bufferMode == mode) return;
active()->bufferMode = mode;
buffers.setBufferMode(active()->path, active()->bufferMode);
if (active()->bufferMode == BufferManager::BufferMode::Text) {
suggestions.clear();
whetstoneDiagnostics.clear();
analysisPending = false;
} else {
onTextChanged();
}
}
bool hasSidePanels() const {
if (ui.showOutline) return true;
if (library.showDependencyPanel) return true;
@@ -2381,6 +2409,25 @@ struct EditorState {
void doOpen(const std::string& path,
BufferManager::BufferMode modeOverride = BufferManager::BufferMode::Structured) {
std::error_code sizeErr;
size_t fileSizeBytes = 0;
if (!path.empty()) {
auto bytes = std::filesystem::file_size(path, sizeErr);
if (!sizeErr) fileSizeBytes = static_cast<size_t>(bytes);
}
const size_t mb = 1024 * 1024;
const size_t warnBytes = (size_t)std::max(1, settings.getLargeFileWarnMB()) * mb;
const size_t textBytes = (size_t)std::max(1, settings.getLargeFileTextMB()) * mb;
const size_t disableHlBytes =
(size_t)std::max(1, settings.getLargeFileDisableHighlightMB()) * mb;
bool warnLarge = fileSizeBytes >= warnBytes;
bool autoText = fileSizeBytes >= textBytes;
bool disableHighlight = fileSizeBytes >= disableHlBytes;
bool largeFileMode = disableHighlight;
BufferManager::BufferMode effectiveMode = modeOverride;
if (autoText && modeOverride == BufferManager::BufferMode::Structured) {
effectiveMode = BufferManager::BufferMode::Text;
}
std::ifstream in(path);
if (in.is_open()) {
std::ostringstream ss;
@@ -2407,9 +2454,19 @@ struct EditorState {
language = "go";
else if (path.size() > 4 && path.substr(path.size() - 4) == ".org")
language = "org";
createBuffer(path, content, language, modeOverride);
welcome.addRecentFile(path, language, bufferModeToString(modeOverride));
createBuffer(path, content, language, effectiveMode,
fileSizeBytes, largeFileMode, disableHighlight);
welcome.addRecentFile(path, language, bufferModeToString(effectiveMode));
saveRecentFiles();
if (autoText && modeOverride == BufferManager::BufferMode::Structured) {
notify(NotificationLevel::Warning,
"Large file opened in Text mode (" +
std::to_string(fileSizeBytes / mb) + " MB).");
} else if (warnLarge && modeOverride == BufferManager::BufferMode::Structured) {
showLargeFilePrompt = true;
largeFilePromptPath = path;
largeFilePromptBytes = fileSizeBytes;
}
notify(NotificationLevel::Success, "Opened: " + path);
} else {
notify(NotificationLevel::Error, "Error opening: " + path);
@@ -2454,6 +2511,11 @@ struct EditorState {
void updateHighlights() {
if (!active()) return;
if (!active()->highlightsDirty) return;
if (active()->disableSyntaxHighlight) {
active()->highlights.clear();
active()->highlightsDirty = false;
return;
}
active()->highlights = SyntaxHighlighter::highlight(active()->editBuf, active()->language);
active()->highlightsDirty = false;
}
@@ -2467,6 +2529,11 @@ struct EditorState {
active()->generatedBuf = generated;
active()->generatedHighlightsDirty = true;
}
if (active()->disableSyntaxHighlight) {
active()->generatedHighlights.clear();
active()->generatedHighlightsDirty = false;
return;
}
if (active()->generatedHighlightsDirty) {
active()->generatedHighlights =
SyntaxHighlighter::highlight(active()->generatedBuf, active()->generatedLanguage);

15
editor/src/SettingsManager.h
View File

@@ -62,6 +62,12 @@ public:
void setUseAnnotationShapes(bool value) { useAnnotationShapes_ = value; }
bool getBlockVulnerableImports() const { return blockVulnerableImports_; }
void setBlockVulnerableImports(bool value) { blockVulnerableImports_ = value; }
int getLargeFileWarnMB() const { return largeFileWarnMB_; }
void setLargeFileWarnMB(int value) { largeFileWarnMB_ = value; }
int getLargeFileTextMB() const { return largeFileTextMB_; }
void setLargeFileTextMB(int value) { largeFileTextMB_ = value; }
int getLargeFileDisableHighlightMB() const { return largeFileDisableHighlightMB_; }
void setLargeFileDisableHighlightMB(int value) { largeFileDisableHighlightMB_ = value; }
const std::string& getLayoutPreset() const { return layoutPreset_; }
void setLayoutPreset(const std::string& name) { layoutPreset_ = name; }
const std::string& getKeybindingProfile() const { return keybindingProfile_; }
@@ -89,6 +95,9 @@ public:
reduceMotion_ = j.value("reduceMotion", reduceMotion_);
useAnnotationShapes_ = j.value("useAnnotationShapes", useAnnotationShapes_);
blockVulnerableImports_ = j.value("blockVulnerableImports", blockVulnerableImports_);
largeFileWarnMB_ = j.value("largeFileWarnMB", largeFileWarnMB_);
largeFileTextMB_ = j.value("largeFileTextMB", largeFileTextMB_);
largeFileDisableHighlightMB_ = j.value("largeFileDisableHighlightMB", largeFileDisableHighlightMB_);
layoutPreset_ = j.value("layoutPreset", layoutPreset_);
keybindingProfile_ = j.value("keybindingProfile", keybindingProfile_);
emacsConfigPath_ = j.value("emacsConfigPath", emacsConfigPath_);
@@ -129,6 +138,9 @@ public:
j["reduceMotion"] = reduceMotion_;
j["useAnnotationShapes"] = useAnnotationShapes_;
j["blockVulnerableImports"] = blockVulnerableImports_;
j["largeFileWarnMB"] = largeFileWarnMB_;
j["largeFileTextMB"] = largeFileTextMB_;
j["largeFileDisableHighlightMB"] = largeFileDisableHighlightMB_;
j["layoutPreset"] = layoutPreset_;
j["keybindingProfile"] = keybindingProfile_;
j["emacsConfigPath"] = emacsConfigPath_;
@@ -206,6 +218,9 @@ private:
bool reduceMotion_ = false;
bool useAnnotationShapes_ = true;
bool blockVulnerableImports_ = false;
int largeFileWarnMB_ = 1;
int largeFileTextMB_ = 5;
int largeFileDisableHighlightMB_ = 10;
std::string layoutPreset_ = "VSCode";
std::string keybindingProfile_ = "VSCode";

1
editor/src/main.cpp
View File

@@ -379,6 +379,7 @@ int main(int, char**) {
renderProjectSearchPanel(state);
renderSettingsPanel(state);
renderGoToLineDialog(state);
renderLargeFilePrompt(state);
renderLspSettingsPanel(state);
renderRefactorPopup(state);
renderCommandPalette(state);

37
editor/src/panels/DialogPanels.h
View File

@@ -3,6 +3,7 @@
#include "../EditorUtils.h"
#include <filesystem>
#include <unordered_set>
#include <cstdio>
static int buildCommandContextMask(const EditorState& state) {
int mask = CommandContext_Editor;
@@ -31,6 +32,14 @@ static std::string makeRelativePath(const std::string& path, const std::string&
return rel.generic_string();
}
static std::string formatBytes(size_t bytes) {
const double mb = 1024.0 * 1024.0;
double value = bytes / mb;
char buf[64];
std::snprintf(buf, sizeof(buf), "%.2f MB", value);
return buf;
}
static void drawHighlightedText(ImDrawList* drawList,
const std::string& text,
const std::vector<int>& indices,
@@ -525,3 +534,31 @@ static void renderCommandPalette(EditorState& state) {
}
ImGui::End();
}
static void renderLargeFilePrompt(EditorState& state) {
if (state.showLargeFilePrompt) {
ImGui::OpenPopup("Large File");
}
if (ImGui::BeginPopupModal("Large File",
&state.showLargeFilePrompt,
ImGuiWindowFlags_AlwaysAutoResize)) {
ImGui::TextUnformatted("Large file detected.");
if (!state.largeFilePromptPath.empty()) {
ImGui::TextWrapped("%s", state.largeFilePromptPath.c_str());
}
ImGui::TextDisabled("Size: %s", formatBytes(state.largeFilePromptBytes).c_str());
ImGui::Separator();
ImGui::TextUnformatted("Open in Text Mode for better performance?");
if (ImGui::Button("Open in Text Mode")) {
state.setActiveBufferMode(BufferManager::BufferMode::Text);
state.showLargeFilePrompt = false;
ImGui::CloseCurrentPopup();
}
ImGui::SameLine();
if (ImGui::Button("Keep Structured")) {
state.showLargeFilePrompt = false;
ImGui::CloseCurrentPopup();
}
ImGui::EndPopup();
}
}

21
editor/src/panels/SettingsPanel.h
View File

@@ -360,6 +360,27 @@ static void renderSettingsPanel(EditorState& state) {
settingsChanged = true;
}
ImGui::Separator();
ImGui::TextUnformatted("Large File Thresholds (MB)");
int warnMb = state.settings.getLargeFileWarnMB();
if (ImGui::InputInt("Warn > MB", &warnMb)) {
warnMb = std::max(1, warnMb);
state.settings.setLargeFileWarnMB(warnMb);
settingsChanged = true;
}
int textMb = state.settings.getLargeFileTextMB();
if (ImGui::InputInt("Auto Text Mode > MB", &textMb)) {
textMb = std::max(1, textMb);
state.settings.setLargeFileTextMB(textMb);
settingsChanged = true;
}
int disableHlMb = state.settings.getLargeFileDisableHighlightMB();
if (ImGui::InputInt("Disable Highlight > MB", &disableHlMb)) {
disableHlMb = std::max(1, disableHlMb);
state.settings.setLargeFileDisableHighlightMB(disableHlMb);
settingsChanged = true;
}
LayoutPreset preset = state.ui.layoutPreset;
int presetIndex = 0;
if (preset == LayoutPreset::Emacs) presetIndex = 1;

58
editor/src/panels/StatusBarPanel.h
View File

@@ -4,6 +4,12 @@
#include "../ThemeEngine.h"
#include <fstream>
#include <filesystem>
#include <cstdio>
#include <string>
#ifdef _WIN32
#include <windows.h>
#include <psapi.h>
#endif
static std::string detectGitBranch(const std::string& root) {
if (root.empty()) return "-";
@@ -22,17 +28,11 @@ static std::string detectGitBranch(const std::string& root) {
static void toggleBufferMode(EditorState& state) {
if (!state.active()) return;
state.active()->bufferMode = state.active()->bufferMode == BufferManager::BufferMode::Text
? BufferManager::BufferMode::Structured
: BufferManager::BufferMode::Text;
state.buffers.setBufferMode(state.active()->path, state.active()->bufferMode);
if (state.active()->bufferMode == BufferManager::BufferMode::Text) {
state.suggestions.clear();
state.whetstoneDiagnostics.clear();
state.analysisPending = false;
} else {
state.onTextChanged();
}
BufferManager::BufferMode next =
state.active()->bufferMode == BufferManager::BufferMode::Text
? BufferManager::BufferMode::Structured
: BufferManager::BufferMode::Text;
state.setActiveBufferMode(next);
}
static std::string detectLineEnding(const std::string& text) {
@@ -40,6 +40,27 @@ static std::string detectLineEnding(const std::string& text) {
return "LF";
}
static size_t processMemoryBytes() {
#ifdef _WIN32
PROCESS_MEMORY_COUNTERS_EX pmc{};
if (GetProcessMemoryInfo(GetCurrentProcess(),
reinterpret_cast<PROCESS_MEMORY_COUNTERS*>(&pmc),
sizeof(pmc))) {
return static_cast<size_t>(pmc.WorkingSetSize);
}
#endif
return 0;
}
static std::string formatMemory(size_t bytes) {
if (bytes == 0) return "-";
const double mb = 1024.0 * 1024.0;
double value = bytes / mb;
char buf[64];
std::snprintf(buf, sizeof(buf), "%.0f MB", value);
return buf;
}
static void renderStatusBar(EditorState& state) {
const ImGuiViewport* viewport = ImGui::GetMainViewport();
ImGuiWindowFlags sbFlags = ImGuiWindowFlags_NoDecoration |
@@ -77,6 +98,14 @@ static void renderStatusBar(EditorState& state) {
if (ImGui::Button(modeLabel.c_str())) {
toggleBufferMode(state);
}
if (state.active() &&
state.active()->bufferMode == BufferManager::BufferMode::Text &&
state.active()->fileSizeBytes > 0) {
ImGui::SameLine(0, 6);
if (ImGui::Button("Reopen Structured")) {
state.setActiveBufferMode(BufferManager::BufferMode::Structured);
}
}
ImGui::SameLine(0, 10);
const char* langLabel = state.active() ? state.active()->language.c_str() : "-";
if (ImGui::Button(langLabel)) {
@@ -140,6 +169,13 @@ static void renderStatusBar(EditorState& state) {
rightText += " | Sel " + std::to_string(selChars) + "c " +
std::to_string(selLines) + "l";
}
if (state.active() && state.active()->largeFileMode) {
rightText += " | Large File";
}
size_t memBytes = processMemoryBytes();
if (memBytes > 0) {
rightText += " | Mem " + formatMemory(memBytes);
}
rightText += " | Zoom " + std::to_string(zoomPercent(state.settings.getFontSize(),
state.baseFontSize)) + "%";
rightText += " | Git " + detectGitBranch(state.workspaceRoot);

40
editor/tests/step199_test.cpp Normal file
View File

@@ -0,0 +1,40 @@
// Step 199: Large file handling.
#include <cassert>
#include <fstream>
#include <string>
static std::string readFile(const std::string& path) {
std::ifstream f(path);
if (!f.is_open()) return {};
return std::string((std::istreambuf_iterator<char>(f)),
std::istreambuf_iterator<char>());
}
static void assertContains(const std::string& text, const std::string& needle) {
assert(text.find(needle) != std::string::npos);
}
int main() {
const std::string settings = readFile("src/SettingsManager.h");
const std::string editorState = readFile("src/EditorState.h");
const std::string dialogs = readFile("src/panels/DialogPanels.h");
const std::string status = readFile("src/panels/StatusBarPanel.h");
assertContains(settings, "largeFileWarnMB");
assertContains(settings, "largeFileTextMB");
assertContains(settings, "largeFileDisableHighlightMB");
assertContains(editorState, "disableSyntaxHighlight");
assertContains(editorState, "largeFilePrompt");
assertContains(editorState, "setActiveBufferMode");
assertContains(dialogs, "Large File");
assertContains(dialogs, "Open in Text Mode");
assertContains(status, "Large File");
assertContains(status, "Mem ");
printf("step199_test: all assertions passed\n");
return 0;
}

2
sprint6_plan.md
View File

@@ -414,7 +414,7 @@ Make Whetstone usable by everyone and fast with large files.
- Benchmark: 50k line file opens in <500ms, scrolls at 60fps
*Modifies:* `CodeEditorWidget.h`
- [ ] **Step 199: Large file handling**
- [x] **Step 199: Large file handling**
Graceful degradation for very large files:
- Files >1MB: show size warning, offer "Open in Text Mode"
- Files >5MB: auto-open in Text Mode (no AST sync, no annotations)