diff --git a/services/web/frontend/js/features/source-editor/extensions/index.ts b/services/web/frontend/js/features/source-editor/extensions/index.ts
index 120734af0a..69ed4125c1 100644
--- a/services/web/frontend/js/features/source-editor/extensions/index.ts
+++ b/services/web/frontend/js/features/source-editor/extensions/index.ts
@@ -112,7 +112,7 @@ export const createExtensions = (options: Record<string, any>): Extension[] => [
   theme(options.theme),
   realtime(options.currentDoc, options.handleError),
   cursorPosition(options.currentDoc),
-  scrollPosition(options.currentDoc),
+  scrollPosition(options.currentDoc, options.visual),
   cursorHighlights(),
   autoPair(options.settings),
   editable(),
diff --git a/services/web/frontend/js/features/source-editor/extensions/scroll-position.ts b/services/web/frontend/js/features/source-editor/extensions/scroll-position.ts
index 98bea6c7ad..3750b3c455 100644
--- a/services/web/frontend/js/features/source-editor/extensions/scroll-position.ts
+++ b/services/web/frontend/js/features/source-editor/extensions/scroll-position.ts
@@ -7,7 +7,7 @@ import {
   Text,
   TransactionSpec,
 } from '@codemirror/state'
-import { toggleVisualEffect } from './visual/visual'
+import { sourceOnly, toggleVisualEffect } from './visual/visual'
 import { debugConsole } from '@/utils/debugging'
 
 const buildStorageKey = (docId: string) => `doc.position.${docId}`
@@ -23,11 +23,14 @@ type LineInfo = {
  *    or the window is closed, or when switching between Source and Rich Text, and
  * b) dispatches the scroll position (middle visible line) when it changes, for use in the outline.
  */
-export const scrollPosition = ({
-  currentDoc: { doc_id: docId },
-}: {
-  currentDoc: { doc_id: string }
-}) => {
+export const scrollPosition = (
+  {
+    currentDoc: { doc_id: docId },
+  }: {
+    currentDoc: { doc_id: string }
+  },
+  { visual }: { visual: boolean }
+) => {
   // store lineInfo for use on unload, when the DOM has already been unmounted
   let lineInfo: LineInfo
 
@@ -90,6 +93,26 @@ export const scrollPosition = ({
         },
       }
     ),
+
+    // restore the scroll position when switching to source mode
+    sourceOnly(
+      visual,
+      EditorView.updateListener.of(update => {
+        for (const tr of update.transactions) {
+          for (const effect of tr.effects) {
+            if (effect.is(toggleVisualEffect)) {
+              if (!effect.value) {
+                // switching to the source editor
+                window.setTimeout(() => {
+                  update.view.dispatch(restoreScrollPosition())
+                  update.view.focus()
+                })
+              }
+            }
+          }
+        }
+      })
+    ),
   ]
 }
 
diff --git a/services/web/frontend/js/features/source-editor/extensions/visual/visual.ts b/services/web/frontend/js/features/source-editor/extensions/visual/visual.ts
index b59410ccb2..e644910c59 100644
--- a/services/web/frontend/js/features/source-editor/extensions/visual/visual.ts
+++ b/services/web/frontend/js/features/source-editor/extensions/visual/visual.ts
@@ -85,23 +85,6 @@ export const sourceOnly = (visual: boolean, extension: Extension) => {
       }
       return null
     }),
-
-    // restore the scroll position when switching to source mode
-    EditorView.updateListener.of(update => {
-      for (const tr of update.transactions) {
-        for (const effect of tr.effects) {
-          if (effect.is(toggleVisualEffect)) {
-            if (!effect.value) {
-              // switching to the source editor
-              window.setTimeout(() => {
-                update.view.dispatch(restoreScrollPosition())
-                update.view.focus()
-              })
-            }
-          }
-        }
-      }
-    }),
   ]
 }
 
diff --git a/services/web/test/frontend/features/source-editor/extensions/scroll-position.test.ts b/services/web/test/frontend/features/source-editor/extensions/scroll-position.test.ts
index 07147aee60..d5b9b68781 100644
--- a/services/web/test/frontend/features/source-editor/extensions/scroll-position.test.ts
+++ b/services/web/test/frontend/features/source-editor/extensions/scroll-position.test.ts
@@ -66,7 +66,7 @@ describe('CodeMirror scroll position extension', function () {
     const view = new EditorView({
       state: EditorState.create({
         doc,
-        extensions: [scrollPosition({ currentDoc })],
+        extensions: [scrollPosition({ currentDoc }, { visual: false })],
       }),
     })
 
@@ -102,7 +102,7 @@ describe('CodeMirror scroll position extension', function () {
     const view = new EditorView({
       state: EditorState.create({
         doc,
-        extensions: [scrollPosition({ currentDoc })],
+        extensions: [scrollPosition({ currentDoc }, { visual: false })],
       }),
     })
     view.dispatch(restoreScrollPosition())