From 4658449abb0f7aac846612e1c8100cb3226229a0 Mon Sep 17 00:00:00 2001 From: Alf Eaton Date: Fri, 25 Apr 2025 11:22:11 +0100 Subject: [PATCH] Start adding client-side word count (#24892) GitOrigin-RevId: 6c17d7bf7095794c003e17939a8302fc6b059262 --- .../src/Features/Project/ProjectController.js | 1 + .../components/actions-word-count.tsx | 34 +-- .../components/word-count-button.tsx | 44 +++ .../components/word-count-client.tsx | 105 +++++++ .../components/word-count-data.ts | 25 ++ .../components/word-count-error.tsx | 10 + .../components/word-count-loading.tsx | 14 + .../components/word-count-modal-content.jsx | 109 -------- .../components/word-count-modal-content.tsx | 42 +++ ...d-count-modal.jsx => word-count-modal.tsx} | 13 +- .../components/word-count-server.tsx | 48 ++++ .../components/word-counts.tsx | 102 +++++++ .../word-count-modal/hooks/use-word-count.js | 26 -- .../js/features/word-count-modal/utils/api.js | 10 - .../utils/count-words-in-file.ts | 256 ++++++++++++++++++ .../utils/find-preamble-extent.ts | 40 +++ .../word-count-modal/utils/segmenters.ts | 64 +++++ .../stories/word-count-modal.stories.jsx | 63 ----- .../stories/word-count-modal.stories.tsx | 80 ++++++ .../components/word-count-modal.spec.tsx | 111 ++++++++ .../components/word-count-modal.test.jsx | 122 --------- 21 files changed, 949 insertions(+), 370 deletions(-) create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-button.tsx create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-client.tsx create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-data.ts create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-error.tsx create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-loading.tsx delete mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.jsx create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.tsx rename services/web/frontend/js/features/word-count-modal/components/{word-count-modal.jsx => word-count-modal.tsx} (66%) create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx create mode 100644 services/web/frontend/js/features/word-count-modal/components/word-counts.tsx delete mode 100644 services/web/frontend/js/features/word-count-modal/hooks/use-word-count.js delete mode 100644 services/web/frontend/js/features/word-count-modal/utils/api.js create mode 100644 services/web/frontend/js/features/word-count-modal/utils/count-words-in-file.ts create mode 100644 services/web/frontend/js/features/word-count-modal/utils/find-preamble-extent.ts create mode 100644 services/web/frontend/js/features/word-count-modal/utils/segmenters.ts delete mode 100644 services/web/frontend/stories/word-count-modal.stories.jsx create mode 100644 services/web/frontend/stories/word-count-modal.stories.tsx create mode 100644 services/web/test/frontend/features/word-count-modal/components/word-count-modal.spec.tsx delete mode 100644 services/web/test/frontend/features/word-count-modal/components/word-count-modal.test.jsx diff --git a/services/web/app/src/Features/Project/ProjectController.js b/services/web/app/src/Features/Project/ProjectController.js index 791df18390..377ed172b7 100644 --- a/services/web/app/src/Features/Project/ProjectController.js +++ b/services/web/app/src/Features/Project/ProjectController.js @@ -355,6 +355,7 @@ const _ProjectController = { 'paywall-change-compile-timeout', 'overleaf-assist-bundle', 'wf-feature-rebrand', + 'word-count-client', ].filter(Boolean) const getUserValues = async userId => diff --git a/services/web/frontend/js/features/editor-left-menu/components/actions-word-count.tsx b/services/web/frontend/js/features/editor-left-menu/components/actions-word-count.tsx index 3e05c907d6..7832939541 100644 --- a/services/web/frontend/js/features/editor-left-menu/components/actions-word-count.tsx +++ b/services/web/frontend/js/features/editor-left-menu/components/actions-word-count.tsx @@ -1,15 +1,10 @@ import { useState, useCallback } from 'react' -import { useTranslation } from 'react-i18next' -import { useDetachCompileContext as useCompileContext } from '../../../shared/context/detach-compile-context' import WordCountModal from '../../word-count-modal/components/word-count-modal' -import LeftMenuButton from './left-menu-button' import * as eventTracking from '../../../infrastructure/event-tracking' -import OLTooltip from '@/features/ui/components/ol/ol-tooltip' +import { WordCountButton } from '@/features/word-count-modal/components/word-count-button' export default function ActionsWordCount() { const [showModal, setShowModal] = useState(false) - const { pdfUrl } = useCompileContext() - const { t } = useTranslation() const handleShowModal = useCallback(() => { eventTracking.sendMB('left-menu-count') @@ -18,32 +13,7 @@ export default function ActionsWordCount() { return ( <> - {pdfUrl ? ( - - {t('word_count')} - - ) : ( - - {/* OverlayTrigger won't fire unless the child is a non-react html element (e.g div, span) */} -
- - {t('word_count')} - -
- - )} + setShowModal(false)} /> ) diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-button.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-button.tsx new file mode 100644 index 0000000000..1c5f1e9072 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-button.tsx @@ -0,0 +1,44 @@ +import { useDetachCompileContext as useCompileContext } from '@/shared/context/detach-compile-context' +import { useTranslation } from 'react-i18next' +import { isSplitTestEnabled } from '@/utils/splitTestUtils' +import LeftMenuButton from '@/features/editor-left-menu/components/left-menu-button' +import OLTooltip from '@/features/ui/components/ol/ol-tooltip' +import { memo } from 'react' + +export const WordCountButton = memo<{ + handleShowModal: () => void +}>(function WordCountButton({ handleShowModal }) { + const { pdfUrl } = useCompileContext() + const { t } = useTranslation() + + const enabled = pdfUrl || isSplitTestEnabled('word-count-client') + + if (!enabled) { + return ( + + {/* OverlayTrigger won't fire unless the child is a non-react html element (e.g div, span) */} +
+ + {t('word_count')} + +
+ + ) + } + + return ( + + {t('word_count')} + + ) +}) diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-client.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-client.tsx new file mode 100644 index 0000000000..5f2660d4a8 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-client.tsx @@ -0,0 +1,105 @@ +import { FC, useEffect, useMemo, useState } from 'react' +import { WordCountData } from '@/features/word-count-modal/components/word-count-data' +import { WordCountLoading } from '@/features/word-count-modal/components/word-count-loading' +import { WordCountError } from '@/features/word-count-modal/components/word-count-error' +import { useProjectContext } from '@/shared/context/project-context' +import useAbortController from '@/shared/hooks/use-abort-controller' +import { useProjectSettingsContext } from '@/features/editor-left-menu/context/project-settings-context' +import { useEditorManagerContext } from '@/features/ide-react/context/editor-manager-context' +import { useFileTreePathContext } from '@/features/file-tree/contexts/file-tree-path' +import { debugConsole } from '@/utils/debugging' +import { signalWithTimeout } from '@/utils/abort-signal' +import { isMainFile } from '@/features/pdf-preview/util/editor-files' +import { countWordsInFile } from '@/features/word-count-modal/utils/count-words-in-file' +import { WordCounts } from '@/features/word-count-modal/components/word-counts' +import { createSegmenters } from '@/features/word-count-modal/utils/segmenters' + +export const WordCountClient: FC = () => { + const [loading, setLoading] = useState(true) + const [error, setError] = useState(false) + const [data, setData] = useState(null) + const { projectSnapshot, rootDocId } = useProjectContext() + const { spellCheckLanguage } = useProjectSettingsContext() + const { openDocs, currentDocument } = useEditorManagerContext() + const { pathInFolder } = useFileTreePathContext() + + const { signal } = useAbortController() + + const segmenters = useMemo(() => { + return createSegmenters(spellCheckLanguage?.replace(/_/, '-')) + }, [spellCheckLanguage]) + + useEffect(() => { + if (currentDocument && segmenters) { + const countWords = async () => { + await openDocs.awaitBufferedOps(signalWithTimeout(signal, 5000)) + await projectSnapshot.refresh() + + if (signal.aborted) return null + + const currentDocSnapshot = currentDocument.getSnapshot() + const currentRootDocId = isMainFile(currentDocSnapshot) + ? currentDocument.doc_id + : rootDocId + if (!currentRootDocId) return null + + const currentRootDocPath = pathInFolder(currentRootDocId) + if (!currentRootDocPath) return null + + const data: WordCountData = { + encode: 'ascii', + textWords: 0, + textCharacters: 0, + headWords: 0, + headCharacters: 0, + abstractWords: 0, + abstractCharacters: 0, + captionWords: 0, + captionCharacters: 0, + footnoteWords: 0, + footnoteCharacters: 0, + outside: 0, + outsideCharacters: 0, + headers: 0, + elements: 0, + mathInline: 0, + mathDisplay: 0, + errors: 0, + messages: '', + } + + countWordsInFile(data, projectSnapshot, currentRootDocPath, segmenters) + + return data + } + + countWords() + .then(data => { + setData(data) + }) + .catch(error => { + debugConsole.error(error) + setError(true) + }) + .finally(() => { + setLoading(false) + }) + } + }, [ + signal, + openDocs, + projectSnapshot, + segmenters, + currentDocument, + rootDocId, + pathInFolder, + ]) + + return ( + <> + {loading && !error && } + {error && } + {data && } + + ) +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-data.ts b/services/web/frontend/js/features/word-count-modal/components/word-count-data.ts new file mode 100644 index 0000000000..a69dc55d45 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-data.ts @@ -0,0 +1,25 @@ +export type ServerWordCountData = { + encode: string + textWords: number + headWords: number + outside: number + headers: number + elements: number + mathInline: number + mathDisplay: number + errors: number + messages: string +} + +export type WordCountData = ServerWordCountData & { + textCharacters: number + headCharacters: number + captionWords: number + captionCharacters: number + footnoteWords: number + footnoteCharacters: number + abstractWords: number + abstractCharacters: number + // outsideWords: number + outsideCharacters: number +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-error.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-error.tsx new file mode 100644 index 0000000000..18c10ff7d3 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-error.tsx @@ -0,0 +1,10 @@ +import OLNotification from '@/features/ui/components/ol/ol-notification' +import { useTranslation } from 'react-i18next' + +export const WordCountError = () => { + const { t } = useTranslation() + + return ( + + ) +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-loading.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-loading.tsx new file mode 100644 index 0000000000..61a4263b04 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-loading.tsx @@ -0,0 +1,14 @@ +import { Spinner } from 'react-bootstrap-5' +import { useTranslation } from 'react-i18next' + +export const WordCountLoading = () => { + const { t } = useTranslation() + + return ( +
+
+ ) +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.jsx b/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.jsx deleted file mode 100644 index 520b00de6e..0000000000 --- a/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.jsx +++ /dev/null @@ -1,109 +0,0 @@ -import PropTypes from 'prop-types' -import { useTranslation } from 'react-i18next' -import { useProjectContext } from '@/shared/context/project-context' -import { useLocalCompileContext } from '@/shared/context/local-compile-context' -import { useWordCount } from '../hooks/use-word-count' -import { - OLModalBody, - OLModalFooter, - OLModalHeader, - OLModalTitle, -} from '@/features/ui/components/ol/ol-modal' -import OLNotification from '@/features/ui/components/ol/ol-notification' -import OLRow from '@/features/ui/components/ol/ol-row' -import OLCol from '@/features/ui/components/ol/ol-col' -import OLButton from '@/features/ui/components/ol/ol-button' -import { Spinner } from 'react-bootstrap-5' - -// NOTE: this component is only mounted when the modal is open -export default function WordCountModalContent({ handleHide }) { - const { _id: projectId } = useProjectContext() - const { clsiServerId } = useLocalCompileContext() - const { t } = useTranslation() - const { data, error, loading } = useWordCount(projectId, clsiServerId) - - return ( - <> - - {t('word_count')} - - - - {loading && !error && ( -
-
- )} - - {error && ( - - )} - - {data && ( -
- {data.messages && ( - - - {data.messages}

- } - /> - - - )} - - - -
{t('total_words')}:
- - {data.textWords} - - - - -
{t('headers')}:
- - {data.headers} - - - - -
{t('math_inline')}:
- - {data.mathInline} - - - - -
{t('math_display')}:
- - {data.mathDisplay} - -
- )} - - - - - {t('close')} - - - - ) -} - -WordCountModalContent.propTypes = { - handleHide: PropTypes.func.isRequired, -} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.tsx new file mode 100644 index 0000000000..75b53eeeba --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-modal-content.tsx @@ -0,0 +1,42 @@ +import { useTranslation } from 'react-i18next' +import { + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/features/ui/components/ol/ol-modal' +import OLButton from '@/features/ui/components/ol/ol-button' +import { WordCountServer } from './word-count-server' +import { WordCountClient } from './word-count-client' +import { isSplitTestEnabled } from '@/utils/splitTestUtils' + +// NOTE: this component is only mounted when the modal is open +export default function WordCountModalContent({ + handleHide, +}: { + handleHide: () => void +}) { + const { t } = useTranslation() + + return ( + <> + + {t('word_count')} + + + + {isSplitTestEnabled('word-count-client') ? ( + + ) : ( + + )} + + + + + {t('close')} + + + + ) +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-modal.jsx b/services/web/frontend/js/features/word-count-modal/components/word-count-modal.tsx similarity index 66% rename from services/web/frontend/js/features/word-count-modal/components/word-count-modal.jsx rename to services/web/frontend/js/features/word-count-modal/components/word-count-modal.tsx index 81f9685fe0..ea8371978d 100644 --- a/services/web/frontend/js/features/word-count-modal/components/word-count-modal.jsx +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-modal.tsx @@ -1,12 +1,14 @@ -import React from 'react' -import PropTypes from 'prop-types' +import { memo } from 'react' import WordCountModalContent from './word-count-modal-content' import withErrorBoundary from '../../../infrastructure/error-boundary' import OLModal from '@/features/ui/components/ol/ol-modal' -const WordCountModal = React.memo(function WordCountModal({ +const WordCountModal = memo(function WordCountModal({ show, handleHide, +}: { + show: boolean + handleHide: () => void }) { return ( @@ -15,9 +17,4 @@ const WordCountModal = React.memo(function WordCountModal({ ) }) -WordCountModal.propTypes = { - show: PropTypes.bool, - handleHide: PropTypes.func.isRequired, -} - export default withErrorBoundary(WordCountModal) diff --git a/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx b/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx new file mode 100644 index 0000000000..fccd49f8c3 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-count-server.tsx @@ -0,0 +1,48 @@ +import { FC, useEffect, useState } from 'react' +import { ServerWordCountData } from '@/features/word-count-modal/components/word-count-data' +import { WordCountLoading } from '@/features/word-count-modal/components/word-count-loading' +import { WordCountError } from '@/features/word-count-modal/components/word-count-error' +import { useProjectContext } from '@/shared/context/project-context' +import { useLocalCompileContext } from '@/shared/context/local-compile-context' +import useAbortController from '@/shared/hooks/use-abort-controller' +import { getJSON } from '@/infrastructure/fetch-json' +import { debugConsole } from '@/utils/debugging' +import { WordCounts } from '@/features/word-count-modal/components/word-counts' + +export const WordCountServer: FC = () => { + const { _id: projectId } = useProjectContext() + const { clsiServerId } = useLocalCompileContext() + + const [loading, setLoading] = useState(true) + const [error, setError] = useState(false) + const [data, setData] = useState(null) + + const { signal } = useAbortController() + + useEffect(() => { + const url = new URL(`/project/${projectId}/wordcount`, window.location.href) + if (clsiServerId) { + url.searchParams.set('clsiserverid', clsiServerId) + } + + getJSON(url.toString(), { signal }) + .then(data => { + setData(data.texcount) + }) + .catch(error => { + debugConsole.error(error) + setError(true) + }) + .finally(() => { + setLoading(false) + }) + }, [projectId, clsiServerId, signal]) + + return ( + <> + {loading && !error && } + {error && } + {data && } + + ) +} diff --git a/services/web/frontend/js/features/word-count-modal/components/word-counts.tsx b/services/web/frontend/js/features/word-count-modal/components/word-counts.tsx new file mode 100644 index 0000000000..dec4d2e6d8 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/components/word-counts.tsx @@ -0,0 +1,102 @@ +import { + ServerWordCountData, + WordCountData, +} from '@/features/word-count-modal/components/word-count-data' +import { useTranslation } from 'react-i18next' +import { FC } from 'react' +import { Container, Row, Col } from 'react-bootstrap-5' +import OLNotification from '@/features/ui/components/ol/ol-notification' + +export const WordCounts: FC< + | { + data: ServerWordCountData + source: 'server' + } + | { + data: WordCountData + source: 'client' + } +> = ({ data, source }) => { + const { t } = useTranslation() + + return ( + + {data.messages && ( + + + {data.messages}

+ } + /> + + + )} + + {source === 'client' ? ( + <> + + +
Text:
+ + {data.textWords} + + + + +
Headers:
+ + {data.headWords} + + + + +
Captions:
+ + {data.captionWords} + + + + +
Footnotes:
+ + {data.footnoteWords} + + + ) : ( + + +
{t('total_words')}:
+ + {data.textWords} + + )} + + {source === 'server' && ( + <> + + +
{t('headers')}:
+ + {data.headers} + + + + +
{t('math_inline')}:
+ + {data.mathInline} + + + + +
{t('math_display')}:
+ + {data.mathDisplay} + + + )} + + ) +} diff --git a/services/web/frontend/js/features/word-count-modal/hooks/use-word-count.js b/services/web/frontend/js/features/word-count-modal/hooks/use-word-count.js deleted file mode 100644 index 51d6cba8f4..0000000000 --- a/services/web/frontend/js/features/word-count-modal/hooks/use-word-count.js +++ /dev/null @@ -1,26 +0,0 @@ -import useAbortController from '../../../shared/hooks/use-abort-controller' -import { fetchWordCount } from '../utils/api' -import { useEffect, useState } from 'react' - -export function useWordCount(projectId, clsiServerId) { - const [loading, setLoading] = useState(true) - const [error, setError] = useState(false) - const [data, setData] = useState() - - const { signal } = useAbortController() - - useEffect(() => { - fetchWordCount(projectId, clsiServerId, { signal }) - .then(data => { - setData(data.texcount) - }) - .catch(() => { - setError(true) - }) - .finally(() => { - setLoading(false) - }) - }, [signal, clsiServerId, projectId]) - - return { data, error, loading } -} diff --git a/services/web/frontend/js/features/word-count-modal/utils/api.js b/services/web/frontend/js/features/word-count-modal/utils/api.js deleted file mode 100644 index 54818f0338..0000000000 --- a/services/web/frontend/js/features/word-count-modal/utils/api.js +++ /dev/null @@ -1,10 +0,0 @@ -import { getJSON } from '../../../infrastructure/fetch-json' - -export function fetchWordCount(projectId, clsiServerId, options) { - let query = '' - if (clsiServerId) { - query = `?clsiserverid=${clsiServerId}` - } - - return getJSON(`/project/${projectId}/wordcount${query}`, options) -} diff --git a/services/web/frontend/js/features/word-count-modal/utils/count-words-in-file.ts b/services/web/frontend/js/features/word-count-modal/utils/count-words-in-file.ts new file mode 100644 index 0000000000..d9a9154620 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/utils/count-words-in-file.ts @@ -0,0 +1,256 @@ +import { ProjectSnapshot } from '@/infrastructure/project-snapshot' +import { LaTeXLanguage } from '@/features/source-editor/languages/latex/latex-language' +import { WordCountData } from '@/features/word-count-modal/components/word-count-data' +import { NodeType, SyntaxNodeRef } from '@lezer/common' +import { debugConsole } from '@/utils/debugging' +import { findPreambleExtent } from '@/features/word-count-modal/utils/find-preamble-extent' +import { Segmenters } from './segmenters' + +const whiteSpaceRe = /^\s$/ + +type Context = 'text' | 'header' | 'abstract' | 'caption' | 'footnote' + +const counters: Record< + Context, + { + word: keyof WordCountData + character: keyof WordCountData + } +> = { + text: { + word: 'textWords', + character: 'textCharacters', + }, + header: { + word: 'headWords', + character: 'headCharacters', + }, + abstract: { + word: 'abstractWords', + character: 'abstractCharacters', + }, + caption: { + word: 'captionWords', + character: 'captionCharacters', + }, + footnote: { + word: 'footnoteWords', + character: 'footnoteCharacters', + }, +} + +const replacementsMap: Map = new Map([ + // LaTeX commands that create part of a word + ['aa', 'å'], + ['AA', 'Å'], + ['ae', 'æ'], + ['AE', 'Æ'], + ['oe', 'œ'], + ['OE', 'Œ'], + ['o', 'ø'], + ['O', 'Ø'], + ['ss', 'ß'], + ['SS', 'SS'], + ['l', 'ł'], + ['L', 'Ł'], + ['dh', 'ð'], + ['DH', 'Ð'], + ['dj', 'đ'], + ['DJ', 'Ð'], + ['th', 'þ'], + ['TH', 'Þ'], + ['ng', 'ŋ'], + ['NG', 'Ŋ'], + ['i', 'ı'], + ['j', 'ȷ'], + ['_', '_'], + // modifier commands for the character in the arguments + ['H', 'a'], + ['c', 'a'], + ['d', 'a'], + ['k', 'a'], + ['v', 'a'], + // modifier symbols for the subsequent character + ["'", ''], + ['^', ''], + ['"', ''], + ['=', ''], + ['.', ''], +]) + +type TextNode = { + from: number + to: number + text: string + context: Context +} + +export const countWordsInFile = ( + data: WordCountData, + projectSnapshot: ProjectSnapshot, + docPath: string, + segmenters: Segmenters +) => { + debugConsole.log(`Counting words in ${docPath}`) + + const content = projectSnapshot.getDocContents(docPath) // TODO: try with extensions + if (!content) return + + // TODO: language from file extension + const tree = LaTeXLanguage.parser.parse(content) + + let currentContext: Context = 'text' + + const textNodes: TextNode[] = [] + + const iterateNode = (nodeRef: SyntaxNodeRef, context: Context = 'text') => { + const previousContext = currentContext + currentContext = context + const { node } = nodeRef + node.cursor().iterate(childNodeRef => { + // TODO: a better way to iterate only descendants? + if (childNodeRef.node !== node) { + return bodyMatcher(childNodeRef.type)?.(childNodeRef) + } + }) + currentContext = previousContext + } + + const headMatcher = NodeType.match< + (nodeRef: SyntaxNodeRef) => boolean | void + >({ + Title(nodeRef) { + data.headers++ + iterateNode(nodeRef, 'header') + return false + }, + }) + + const bodyMatcher = NodeType.match< + (nodeRef: SyntaxNodeRef) => boolean | void + >({ + Normal(nodeRef) { + textNodes.push({ + from: nodeRef.from, + to: nodeRef.to, + text: content.substring(nodeRef.from, nodeRef.to), + context: currentContext, + }) + }, + Command(nodeRef) { + const child = nodeRef.node.getChild('UnknownCommand') + if (!child) return + + const grandchild = child.getChild('CtrlSeq') ?? child.getChild('CtrlSym') + if (!grandchild) return + + const commandName = content.substring(grandchild.from + 1, grandchild.to) + if (!commandName) return + + if (!replacementsMap.has(commandName)) return + + const text = replacementsMap.get(commandName)! + textNodes.push({ + from: nodeRef.from, + to: nodeRef.to, + text, + context: currentContext, + }) + return false + }, + BeginEnv(nodeRef) { + const envName = content + ?.substring(nodeRef.from + '\\begin{'.length, nodeRef.to - 1) + .replace(/\*$/, '') + + if (envName === 'abstract') { + data.headers++ + iterateNode(nodeRef, 'abstract') + return false + } + }, + 'ShortTextArgument ShortOptionalArg'() { + return false + }, + SectioningArgument(nodeRef) { + data.headers++ + iterateNode(nodeRef, 'header') + return false + }, + 'DisplayMath BracketMath'() { + data.mathDisplay++ + }, + 'InlineMath ParenMath'() { + data.mathInline++ + }, + Caption(nodeRef) { + iterateNode(nodeRef, 'caption') + return false + }, + 'FootnoteCommand EndnoteCommand'(nodeRef) { + iterateNode(nodeRef, 'footnote') + return false + }, + 'IncludeArgument InputArgument'(nodeRef) { + let path = content.substring(nodeRef.from + 1, nodeRef.to - 1) + if (!/\.\w+$/.test(path)) { + path += '.tex' + } + debugConsole.log(path) + if (path) { + countWordsInFile(data, projectSnapshot, path, segmenters) + } + }, + }) + + const preambleExtent = findPreambleExtent(tree) + + tree.iterate({ + from: 0, + to: preambleExtent.to, + enter(nodeRef) { + return headMatcher(nodeRef.type)?.(nodeRef) + }, + }) + + tree.iterate({ + from: preambleExtent.to, + enter(nodeRef) { + return bodyMatcher(nodeRef.type)?.(nodeRef) + }, + }) + + const texts: Record = { + abstract: '', + header: '', + caption: '', + text: '', + footnote: '', + } + + let pos = 0 + for (const textNode of textNodes) { + if (textNode.from !== pos) { + texts[textNode.context] += ' ' + } + texts[textNode.context] += textNode.text + pos = textNode.to + } + + for (const [context, text] of Object.entries(texts)) { + const counter = counters[context as Context] + + for (const value of segmenters.word.segment(text)) { + if (value.isWordLike) { + data[counter.word]++ + } + } + + for (const value of segmenters.character.segment(text)) { + // TODO: option for whether to include whitespace? + if (!whiteSpaceRe.test(value.segment)) { + data[counter.character]++ + } + } + } +} diff --git a/services/web/frontend/js/features/word-count-modal/utils/find-preamble-extent.ts b/services/web/frontend/js/features/word-count-modal/utils/find-preamble-extent.ts new file mode 100644 index 0000000000..21aafe4b96 --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/utils/find-preamble-extent.ts @@ -0,0 +1,40 @@ +import { NodeType, SyntaxNodeRef, Tree } from '@lezer/common' +import { ancestorOfNodeWithType } from '@/features/source-editor/utils/tree-operations/ancestors' + +export const findPreambleExtent = (tree: Tree) => { + const preamble = { to: 0 } + + let seenDocumentEnvironment = false + + const preambleMatcher = NodeType.match<(nodeRef: SyntaxNodeRef) => void>({ + 'Title Author Affil Affiliation'(nodeRef) { + preamble.to = nodeRef.node.to + }, + DocumentEnvironment(nodeRef) { + // only count the first instance of DocumentEnvironment + if (!seenDocumentEnvironment) { + preamble.to = + nodeRef.node.getChild('Content')?.from ?? nodeRef.node.from + seenDocumentEnvironment = true + } + }, + Maketitle(nodeRef) { + // count \maketitle inside DocumentEnvironment + if ( + ancestorOfNodeWithType(nodeRef.node, '$Environment')?.type.is( + 'DocumentEnvironment' + ) + ) { + preamble.to = nodeRef.node.from + } + }, + }) + + tree.iterate({ + enter(nodeRef) { + return preambleMatcher(nodeRef.type)?.(nodeRef) + }, + }) + + return preamble +} diff --git a/services/web/frontend/js/features/word-count-modal/utils/segmenters.ts b/services/web/frontend/js/features/word-count-modal/utils/segmenters.ts new file mode 100644 index 0000000000..e14c3365fd --- /dev/null +++ b/services/web/frontend/js/features/word-count-modal/utils/segmenters.ts @@ -0,0 +1,64 @@ +const wordRe = /['\-.\p{L}]+/gu +const wordLikeRe = /\p{L}/gu // must contain at least one "letter" to be a word +const characterRe = /\S/gu + +type SegmentDataLike = { + segment: string + isWordLike?: boolean +} + +type SegmenterLike = { + segment(input: string): { + [Symbol.iterator](): IterableIterator + } +} + +export type Segmenters = { + word: SegmenterLike + character: SegmenterLike +} + +export const createSegmenters = (locale?: string): Segmenters => { + if (!Intl.Segmenter) { + return genericSegmenters + } + + try { + return { + word: new Intl.Segmenter(locale, { + granularity: 'word', // TODO: count hyphenated words as a single word + }), + character: new Intl.Segmenter(locale, { + granularity: 'grapheme', + }), + } + } catch { + return genericSegmenters + } +} + +const genericSegmenters = { + word: { + segment(input: string) { + const segments: SegmentDataLike[] = [] + for (const match of input.matchAll(wordRe)) { + segments.push({ + segment: match[0], + isWordLike: wordLikeRe.test(match[0]), + }) + } + return segments + }, + }, + character: { + segment(input: string) { + const segments: SegmentDataLike[] = [] + for (const match of input.matchAll(characterRe)) { + segments.push({ + segment: match[0], + }) + } + return segments + }, + }, +} diff --git a/services/web/frontend/stories/word-count-modal.stories.jsx b/services/web/frontend/stories/word-count-modal.stories.jsx deleted file mode 100644 index 7742815ea6..0000000000 --- a/services/web/frontend/stories/word-count-modal.stories.jsx +++ /dev/null @@ -1,63 +0,0 @@ -import useFetchMock from './hooks/use-fetch-mock' -import WordCountModal from '../js/features/word-count-modal/components/word-count-modal' -import { ScopeDecorator } from './decorators/scope' - -const counts = { - headers: 4, - mathDisplay: 40, - mathInline: 400, - textWords: 4000, -} - -const messages = [ - 'Lorem ipsum dolor sit amet.', - 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.', -].join('\n') - -export const WordCount = args => { - useFetchMock(fetchMock => { - fetchMock.get( - 'express:/project/:projectId/wordcount', - { status: 200, body: { texcount: counts } }, - { delay: 500 } - ) - }) - - return -} - -export const WordCountWithMessages = args => { - useFetchMock(fetchMock => { - fetchMock.get( - 'express:/project/:projectId/wordcount', - { status: 200, body: { texcount: { ...counts, messages } } }, - { delay: 500 } - ) - }) - - return -} - -export const ErrorResponse = args => { - useFetchMock(fetchMock => { - fetchMock.get( - 'express:/project/:projectId/wordcount', - { status: 500 }, - { delay: 500 } - ) - }) - - return -} - -export default { - title: 'Editor / Modals / Word Count', - component: WordCountModal, - args: { - show: true, - }, - argTypes: { - handleHide: { action: 'close modal' }, - }, - decorators: [ScopeDecorator], -} diff --git a/services/web/frontend/stories/word-count-modal.stories.tsx b/services/web/frontend/stories/word-count-modal.stories.tsx new file mode 100644 index 0000000000..17b548791b --- /dev/null +++ b/services/web/frontend/stories/word-count-modal.stories.tsx @@ -0,0 +1,80 @@ +import { Meta, StoryObj } from '@storybook/react' +import WordCountModal from '@/features/word-count-modal/components/word-count-modal' +import { ScopeDecorator } from './decorators/scope' +import useFetchMock from './hooks/use-fetch-mock' + +export default { + title: 'Editor / Modals / Word Count', + component: WordCountModal, + args: { + show: true, + }, + argTypes: { + handleHide: { + action: 'close modal', + }, + }, + decorators: [Story => ScopeDecorator(Story)], +} satisfies Meta + +type Story = StoryObj + +const counts = { + headers: 4, + mathDisplay: 40, + mathInline: 400, + textWords: 4000, +} + +const messages = [ + 'Lorem ipsum dolor sit amet.', + 'Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.', +].join('\n') + +export const WordCount: Story = { + decorators: [ + Story => { + useFetchMock(fetchMock => { + fetchMock.get( + 'express:/project/:projectId/wordcount', + { status: 200, body: { texcount: counts } }, + { delay: 500 } + ) + }) + + return + }, + ], +} + +export const WordCountWithMessages: Story = { + decorators: [ + Story => { + useFetchMock(fetchMock => { + fetchMock.get( + 'express:/project/:projectId/wordcount', + { status: 200, body: { texcount: { ...counts, messages } } }, + { delay: 500 } + ) + }) + + return + }, + ], +} + +export const ErrorResponse: Story = { + decorators: [ + Story => { + useFetchMock(fetchMock => { + fetchMock.get( + 'express:/project/:projectId/wordcount', + { status: 500 }, + { delay: 500 } + ) + }) + + return + }, + ], +} diff --git a/services/web/test/frontend/features/word-count-modal/components/word-count-modal.spec.tsx b/services/web/test/frontend/features/word-count-modal/components/word-count-modal.spec.tsx new file mode 100644 index 0000000000..2ed1dc9448 --- /dev/null +++ b/services/web/test/frontend/features/word-count-modal/components/word-count-modal.spec.tsx @@ -0,0 +1,111 @@ +import WordCountModal from '@/features/word-count-modal/components/word-count-modal' +import { EditorProviders } from '../../../helpers/editor-providers' + +describe('', function () { + beforeEach(function () { + cy.interceptCompile() + }) + + it('renders the translated modal title', function () { + cy.intercept('/project/*/wordcount*', { + body: { texcount: { messages: 'This is a test' } }, + }) + + cy.mount( + + + + ) + + cy.findByText('Word Count') + cy.findByText(/something went wrong/).should('not.exist') + }) + + it('renders a loading message when loading', function () { + const { promise, resolve } = Promise.withResolvers() + + cy.intercept('/project/*/wordcount*', async req => { + await promise + req.reply({ texcount: { messages: 'This is a test' } }) + }) + + cy.mount( + + + + ) + + cy.findByText('Loading…').then(() => { + resolve() + }) + + cy.findByText('This is a test') + }) + + it('renders an error message and hides loading message on error', function () { + cy.intercept('/project/*/wordcount?*', { + statusCode: 500, + }) + + cy.mount( + + + + ) + + cy.findByText('Sorry, something went wrong') + + cy.findByText('Loading').should('not.exist') + }) + + it('displays messages', function () { + cy.intercept('/project/*/wordcount*', { + body: { texcount: { messages: 'This is a test' } }, + }) + + cy.mount( + + + + ) + + cy.findByText('This is a test') + }) + + it('displays counts data', function () { + cy.intercept('/project/*/wordcount*', { + body: { + texcount: { + textWords: 500, + headWords: 100, + outside: 200, + mathDisplay: 2, + mathInline: 3, + headers: 4, + }, + }, + }) + + cy.mount( + + + + ) + + cy.findByText((content, element) => { + return /^Total Words\s*:\s*500$/.test(element!.textContent!.trim()) + }) + + cy.findByText((content, element) => { + return /^Math Display\s*:\s*2$/.test(element!.textContent!.trim()) + }) + + cy.findByText((content, element) => { + return /^Math Inline\s*:\s*3$/.test(element!.textContent!.trim()) + }) + + cy.findByText((content, element) => { + return /^Headers\s*:\s*4$/.test(element!.textContent!.trim()) + }) + }) +}) diff --git a/services/web/test/frontend/features/word-count-modal/components/word-count-modal.test.jsx b/services/web/test/frontend/features/word-count-modal/components/word-count-modal.test.jsx deleted file mode 100644 index baff2c2d22..0000000000 --- a/services/web/test/frontend/features/word-count-modal/components/word-count-modal.test.jsx +++ /dev/null @@ -1,122 +0,0 @@ -import { screen } from '@testing-library/react' -import { expect } from 'chai' -import sinon from 'sinon' -import fetchMock from 'fetch-mock' -import { renderWithEditorContext } from '../../../helpers/render-with-context' -import WordCountModal from '../../../../../frontend/js/features/word-count-modal/components/word-count-modal' - -describe('', function () { - afterEach(function () { - fetchMock.removeRoutes().clearHistory() - }) - - const contextProps = { - projectId: 'project-1', - clsiServerId: 'clsi-server-1', - } - - it('renders the translated modal title', async function () { - fetchMock.get('express:/project/:projectId/wordcount', () => { - return { status: 200, body: { texcount: { messages: 'This is a test' } } } - }) - - const handleHide = sinon.stub() - - renderWithEditorContext( - , - contextProps - ) - - await screen.findByText('Word Count') - }) - - it('renders a loading message when loading', async function () { - fetchMock.get('express:/project/:projectId/wordcount', () => { - return { status: 200, body: { texcount: { messages: 'This is a test' } } } - }) - - const handleHide = sinon.stub() - - renderWithEditorContext( - , - contextProps - ) - - await screen.findByText('Loading…') - - await screen.findByText('This is a test') - }) - - it('renders an error message and hides loading message on error', async function () { - fetchMock.get('express:/project/:projectId/wordcount', 500) - - const handleHide = sinon.stub() - - renderWithEditorContext( - , - contextProps - ) - - await screen.findByText('Sorry, something went wrong') - - expect(screen.queryByText(/Loading/)).to.not.exist - }) - - it('displays messages', async function () { - fetchMock.get('express:/project/:projectId/wordcount', () => { - return { - status: 200, - body: { - texcount: { - messages: 'This is a test', - }, - }, - } - }) - - const handleHide = sinon.stub() - - renderWithEditorContext( - , - contextProps - ) - - await screen.findByText('This is a test') - }) - - it('displays counts data', async function () { - fetchMock.get('express:/project/:projectId/wordcount', () => { - return { - status: 200, - body: { - texcount: { - textWords: 100, - mathDisplay: 200, - mathInline: 300, - headers: 400, - }, - }, - } - }) - - const handleHide = sinon.stub() - - renderWithEditorContext( - , - contextProps - ) - - await screen.findByText((content, element) => - element.textContent.trim().match(/^Total Words\s*:\s*100$/) - ) - await screen.findByText((content, element) => - element.textContent.trim().match(/^Math Display\s*:\s*200$/) - ) - await screen.findByText((content, element) => - element.textContent.trim().match(/^Math Inline\s*:\s*300$/) - ) - await screen.findByText((content, element) => - element.textContent.trim().match(/^Headers\s*:\s*400$/) - ) - }) -})