From db3d57255145cfdf930e7dffdbeb14043a673098 Mon Sep 17 00:00:00 2001 From: silverwind Date: Tue, 19 Sep 2023 23:01:24 +0200 Subject: [PATCH 01/12] Fix EOL handling in web editor --- routers/web/repo/editor.go | 2 +- templates/repo/editor/edit.tmpl | 4 ++-- web_src/js/features/codeeditor.js | 17 ++++++++++++++--- web_src/js/utils.js | 11 +++++++++++ web_src/js/utils.test.js | 9 +++++++++ 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go index 0a606582e583b..9d9fee3a77c37 100644 --- a/routers/web/repo/editor.go +++ b/routers/web/repo/editor.go @@ -287,7 +287,7 @@ func editFilePost(ctx *context.Context, form forms.EditRepoFileForm, isNewFile b Operation: operation, FromTreePath: ctx.Repo.TreePath, TreePath: form.TreePath, - ContentReader: strings.NewReader(strings.ReplaceAll(form.Content, "\r", "")), + ContentReader: strings.NewReader(form.Content), }, }, Signoff: form.Signoff, diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index 2b303be97ccdb..490147e263848 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -38,9 +38,9 @@ data-url="{{.Repository.Link}}/markup" data-context="{{.RepoLink}}" data-previewable-extensions="{{.PreviewableExtensions}}" - data-line-wrap-extensions="{{.LineWrapExtensions}}"> -{{.FileContent}} + data-line-wrap-extensions="{{.LineWrapExtensions}}">{{.FileContent}}
+
{{.locale.Tr "loading"}} diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 7dbbcd3dd62a9..e480875f47c86 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -1,5 +1,5 @@ import tinycolor from 'tinycolor2'; -import {basename, extname, isObject, isDarkTheme} from '../utils.js'; +import {basename, extname, isObject, isDarkTheme, detectEol} from '../utils.js'; import {onInputDebounce} from '../utils/dom.js'; const languagesByFilename = {}; @@ -62,7 +62,7 @@ export async function createMonaco(textarea, filename, editorOpts) { const monaco = await import(/* webpackChunkName: "monaco" */'monaco-editor'); initLanguages(monaco); - let {language, ...other} = editorOpts; + let {language, eol, ...other} = editorOpts; if (!language) language = getLanguage(filename); const container = document.createElement('div'); @@ -105,14 +105,23 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); + // TODO: there must be a better way to preserve CRLF in the template rendering + const value = window.monacoContent || ''; + delete window.monacoContent; + textarea.value = value; + const editor = monaco.editor.create(container, { - value: textarea.value, + value, theme: 'gitea', language, ...other, }); const model = editor.getModel(); + + // set eol mode to value from editorconfig or content-detected value + model.setEOL(monaco.editor.EndOfLineSequence[eol || detectEol(value) || 'LF']); + model.onDidChangeContent(() => { textarea.value = editor.getValue(); textarea.dispatchEvent(new Event('change')); // seems to be needed for jquery-are-you-sure @@ -187,5 +196,7 @@ function getEditorConfigOptions(ec) { opts.trimAutoWhitespace = ec.trim_trailing_whitespace === true; opts.insertSpaces = ec.indent_style === 'space'; opts.useTabStops = ec.indent_style === 'tab'; + const eol = ec.end_of_line?.toUpperCase(); + if (['LF', 'CRLF'].includes(eol)) opts.eol = eol; return opts; } diff --git a/web_src/js/utils.js b/web_src/js/utils.js index 1b701e1c6a53f..974261aa22fbb 100644 --- a/web_src/js/utils.js +++ b/web_src/js/utils.js @@ -128,3 +128,14 @@ export function decodeURLEncodedBase64(base64url) { .replace(/_/g, '/') .replace(/-/g, '+')); } + +// Detect dominant line ending in a string +// based on https://github.com/sindresorhus/detect-newline +export function detectEol(str) { + const newlines = (str || '').match(/\r?\n/g) || []; + if (newlines.length === 0) return undefined; + const crlf = newlines.filter((newline) => newline === '\r\n').length; + const lf = newlines.length - crlf; + if (crlf === lf) return undefined; + return crlf > lf ? 'CRLF' : 'LF'; +} diff --git a/web_src/js/utils.test.js b/web_src/js/utils.test.js index db9b1a14a342f..a000537b3b87b 100644 --- a/web_src/js/utils.test.js +++ b/web_src/js/utils.test.js @@ -3,6 +3,7 @@ import { basename, extname, isObject, stripTags, parseIssueHref, parseUrl, translateMonth, translateDay, blobToDataURI, toAbsoluteUrl, encodeURLEncodedBase64, decodeURLEncodedBase64, + detectEol, } from './utils.js'; test('basename', () => { @@ -113,3 +114,11 @@ test('encodeURLEncodedBase64, decodeURLEncodedBase64', () => { expect(Array.from(decodeURLEncodedBase64('YQ'))).toEqual(Array.from(uint8array('a'))); expect(Array.from(decodeURLEncodedBase64('YQ=='))).toEqual(Array.from(uint8array('a'))); }); + +test('detectEol', () => { + expect(detectEol(undefined)).toEqual(undefined); + expect(detectEol('')).toEqual(undefined); + expect(detectEol('a\nb')).toEqual('LF'); + expect(detectEol('a\nb\r\n')).toEqual(undefined); + expect(detectEol('a\nb\r\nc\r\n')).toEqual('CRLF'); +}); From 5566a2da081e6caab6dab2f5d5b1f48bbab8cbeb Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 00:28:14 +0200 Subject: [PATCH 02/12] use monaco's eol detection --- web_src/js/features/codeeditor.js | 11 +++++++---- web_src/js/utils.js | 11 ----------- web_src/js/utils.test.js | 9 --------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index e480875f47c86..4842124dbab88 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -1,5 +1,5 @@ import tinycolor from 'tinycolor2'; -import {basename, extname, isObject, isDarkTheme, detectEol} from '../utils.js'; +import {basename, extname, isObject, isDarkTheme} from '../utils.js'; import {onInputDebounce} from '../utils/dom.js'; const languagesByFilename = {}; @@ -119,8 +119,11 @@ export async function createMonaco(textarea, filename, editorOpts) { const model = editor.getModel(); - // set eol mode to value from editorconfig or content-detected value - model.setEOL(monaco.editor.EndOfLineSequence[eol || detectEol(value) || 'LF']); + // monaco performs auto-detection of dominant EOL in the file, biased towards LF for + // empty files, if there is an editorconfig value, override this detected value + if (eol) { + model.setEOL(monaco.editor.EndOfLineSequence[eol]); + } model.onDidChangeContent(() => { textarea.value = editor.getValue(); @@ -197,6 +200,6 @@ function getEditorConfigOptions(ec) { opts.insertSpaces = ec.indent_style === 'space'; opts.useTabStops = ec.indent_style === 'tab'; const eol = ec.end_of_line?.toUpperCase(); - if (['LF', 'CRLF'].includes(eol)) opts.eol = eol; + opts.eol = ['LF', 'CRLF'].includes(eol) ? eol : undefined; return opts; } diff --git a/web_src/js/utils.js b/web_src/js/utils.js index 974261aa22fbb..1b701e1c6a53f 100644 --- a/web_src/js/utils.js +++ b/web_src/js/utils.js @@ -128,14 +128,3 @@ export function decodeURLEncodedBase64(base64url) { .replace(/_/g, '/') .replace(/-/g, '+')); } - -// Detect dominant line ending in a string -// based on https://github.com/sindresorhus/detect-newline -export function detectEol(str) { - const newlines = (str || '').match(/\r?\n/g) || []; - if (newlines.length === 0) return undefined; - const crlf = newlines.filter((newline) => newline === '\r\n').length; - const lf = newlines.length - crlf; - if (crlf === lf) return undefined; - return crlf > lf ? 'CRLF' : 'LF'; -} diff --git a/web_src/js/utils.test.js b/web_src/js/utils.test.js index a000537b3b87b..db9b1a14a342f 100644 --- a/web_src/js/utils.test.js +++ b/web_src/js/utils.test.js @@ -3,7 +3,6 @@ import { basename, extname, isObject, stripTags, parseIssueHref, parseUrl, translateMonth, translateDay, blobToDataURI, toAbsoluteUrl, encodeURLEncodedBase64, decodeURLEncodedBase64, - detectEol, } from './utils.js'; test('basename', () => { @@ -114,11 +113,3 @@ test('encodeURLEncodedBase64, decodeURLEncodedBase64', () => { expect(Array.from(decodeURLEncodedBase64('YQ'))).toEqual(Array.from(uint8array('a'))); expect(Array.from(decodeURLEncodedBase64('YQ=='))).toEqual(Array.from(uint8array('a'))); }); - -test('detectEol', () => { - expect(detectEol(undefined)).toEqual(undefined); - expect(detectEol('')).toEqual(undefined); - expect(detectEol('a\nb')).toEqual('LF'); - expect(detectEol('a\nb\r\n')).toEqual(undefined); - expect(detectEol('a\nb\r\nc\r\n')).toEqual('CRLF'); -}); From 598a2e60b2cf263d92e44c02cc89b0ee7b439882 Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 00:29:45 +0200 Subject: [PATCH 03/12] comment tweak --- web_src/js/features/codeeditor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 4842124dbab88..a24959b48623a 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -119,8 +119,8 @@ export async function createMonaco(textarea, filename, editorOpts) { const model = editor.getModel(); - // monaco performs auto-detection of dominant EOL in the file, biased towards LF for - // empty files, if there is an editorconfig value, override this detected value + // Monaco performs auto-detection of dominant EOL in the file, biased towards LF for + // empty files. If there is an editorconfig value, override this detected value. if (eol) { model.setEOL(monaco.editor.EndOfLineSequence[eol]); } From c585eaa34ec143bf336e0a2ac35e0028cf1163fa Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 00:35:03 +0200 Subject: [PATCH 04/12] refactor --- web_src/js/features/codeeditor.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index a24959b48623a..32a88cb7a27de 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -121,7 +121,7 @@ export async function createMonaco(textarea, filename, editorOpts) { // Monaco performs auto-detection of dominant EOL in the file, biased towards LF for // empty files. If there is an editorconfig value, override this detected value. - if (eol) { + if (eol in monaco.editor.EndOfLineSequence) { model.setEOL(monaco.editor.EndOfLineSequence[eol]); } @@ -199,7 +199,6 @@ function getEditorConfigOptions(ec) { opts.trimAutoWhitespace = ec.trim_trailing_whitespace === true; opts.insertSpaces = ec.indent_style === 'space'; opts.useTabStops = ec.indent_style === 'tab'; - const eol = ec.end_of_line?.toUpperCase(); - opts.eol = ['LF', 'CRLF'].includes(eol) ? eol : undefined; + opts.eol = ec.end_of_line?.toUpperCase(); return opts; } From 1626c1408f5cdab3528741b32424127bca8c3ddf Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 00:43:54 +0200 Subject: [PATCH 05/12] Update templates/repo/editor/edit.tmpl --- templates/repo/editor/edit.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index 490147e263848..07653d20fd39a 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -38,7 +38,7 @@ data-url="{{.Repository.Link}}/markup" data-context="{{.RepoLink}}" data-previewable-extensions="{{.PreviewableExtensions}}" - data-line-wrap-extensions="{{.LineWrapExtensions}}">{{.FileContent}} + data-line-wrap-extensions="{{.LineWrapExtensions}}">
From f93a1b2fa55546c4d3a071167c4e565ba057e931 Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 21:13:54 +0200 Subject: [PATCH 06/12] illustrate CR issue --- templates/repo/editor/edit.tmpl | 6 ++++-- web_src/js/features/codeeditor.js | 11 +++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index 07653d20fd39a..d9330c7be2854 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -38,9 +38,11 @@ data-url="{{.Repository.Link}}/markup" data-context="{{.RepoLink}}" data-previewable-extensions="{{.PreviewableExtensions}}" - data-line-wrap-extensions="{{.LineWrapExtensions}}"> + data-value="{{.FileContent}}" + data="{{.FileContent}}" + data-foo="{{.FileContent}}" + data-line-wrap-extensions="{{.LineWrapExtensions}}">{{.FileContent}}
-
{{.locale.Tr "loading"}} diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 32a88cb7a27de..4f11e6072d3a8 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -105,18 +105,17 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); - // TODO: there must be a better way to preserve CRLF in the template rendering - const value = window.monacoContent || ''; - delete window.monacoContent; - textarea.value = value; - const editor = monaco.editor.create(container, { - value, + value: textarea.value, theme: 'gitea', language, ...other, }); + console.log(JSON.stringify(textarea.value)); + console.log(JSON.stringify(textarea.getAttribute('data'))); + console.log(JSON.stringify(textarea.getAttribute('data-foo'))); + const model = editor.getModel(); // Monaco performs auto-detection of dominant EOL in the file, biased towards LF for From d6fd3f6cfeabfaa0b3fd1a724dbbb47e98f694d5 Mon Sep 17 00:00:00 2001 From: silverwind Date: Wed, 20 Sep 2023 21:28:28 +0200 Subject: [PATCH 07/12] add devtest showcase --- templates/devtest/gitea-ui.tmpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/devtest/gitea-ui.tmpl b/templates/devtest/gitea-ui.tmpl index 258b72f8cd5f0..93014de6f9212 100644 --- a/templates/devtest/gitea-ui.tmpl +++ b/templates/devtest/gitea-ui.tmpl @@ -1,6 +1,7 @@ {{template "base/head" .}} -
+
+

Button

From f394554ba9aa862c7cbd27e6592a46d0476b2309 Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 21 Sep 2023 11:54:11 +0200 Subject: [PATCH 08/12] encode with json, remove debugs --- templates/devtest/gitea-ui.tmpl | 3 +-- templates/repo/editor/edit.tmpl | 6 ++---- web_src/js/features/codeeditor.js | 12 +++++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/templates/devtest/gitea-ui.tmpl b/templates/devtest/gitea-ui.tmpl index 93014de6f9212..258b72f8cd5f0 100644 --- a/templates/devtest/gitea-ui.tmpl +++ b/templates/devtest/gitea-ui.tmpl @@ -1,7 +1,6 @@ {{template "base/head" .}} -
- +

Button

diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index d9330c7be2854..759e46ce7848e 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -38,10 +38,8 @@ data-url="{{.Repository.Link}}/markup" data-context="{{.RepoLink}}" data-previewable-extensions="{{.PreviewableExtensions}}" - data-value="{{.FileContent}}" - data="{{.FileContent}}" - data-foo="{{.FileContent}}" - data-line-wrap-extensions="{{.LineWrapExtensions}}">{{.FileContent}} + data-initial-value="{{JsonUtils.EncodeToString .FileContent}}" + data-line-wrap-extensions="{{.LineWrapExtensions}}">
diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 4f11e6072d3a8..826dbc2fd551b 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -105,17 +105,19 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); + // The initial is encoded in JSON by the backend to prevent browsers from + // discarding \r during HTML parsing. + const value = JSON.parse(textarea.getAttribute('data-initial-value') || '""'); + textarea.value = value; + textarea.removeAttribute('data-initial-value'); + const editor = monaco.editor.create(container, { - value: textarea.value, + value, theme: 'gitea', language, ...other, }); - console.log(JSON.stringify(textarea.value)); - console.log(JSON.stringify(textarea.getAttribute('data'))); - console.log(JSON.stringify(textarea.getAttribute('data-foo'))); - const model = editor.getModel(); // Monaco performs auto-detection of dominant EOL in the file, biased towards LF for From 60369e08f8f86331438352de1c0b91bd1556ea5a Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 21 Sep 2023 11:57:10 +0200 Subject: [PATCH 09/12] fix comment --- web_src/js/features/codeeditor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 826dbc2fd551b..3d2be09a45b88 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -105,7 +105,7 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); - // The initial is encoded in JSON by the backend to prevent browsers from + // The initial value is encoded in JSON by the backend to prevent browsers from // discarding \r during HTML parsing. const value = JSON.parse(textarea.getAttribute('data-initial-value') || '""'); textarea.value = value; From 455732943bd9f1dce140c3de73b8233cc0ec258d Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 21 Sep 2023 12:00:39 +0200 Subject: [PATCH 10/12] update comment --- web_src/js/features/codeeditor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 3d2be09a45b88..06d29985c55e0 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -105,8 +105,8 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); - // The initial value is encoded in JSON by the backend to prevent browsers from - // discarding \r during HTML parsing. + // We encode the initial value in JSON on the backend to prevent browsers from discarding + // \r during HTML parsing as per https://infra.spec.whatwg.org/#normalize-newlines. const value = JSON.parse(textarea.getAttribute('data-initial-value') || '""'); textarea.value = value; textarea.removeAttribute('data-initial-value'); From 8545d159541be9d5bd2ccda22cbcc9c1b8cce306 Mon Sep 17 00:00:00 2001 From: silverwind Date: Thu, 21 Sep 2023 12:05:33 +0200 Subject: [PATCH 11/12] better link --- web_src/js/features/codeeditor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 06d29985c55e0..5f924fd0864cf 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -105,8 +105,9 @@ export async function createMonaco(textarea, filename, editorOpts) { monaco.languages.register({id: 'vs.editor.nullLanguage'}); monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); - // We encode the initial value in JSON on the backend to prevent browsers from discarding - // \r during HTML parsing as per https://infra.spec.whatwg.org/#normalize-newlines. + // We encode the initial value in JSON on the backend to prevent browsers from + // discarding the \r during HTML parsing: + // https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream const value = JSON.parse(textarea.getAttribute('data-initial-value') || '""'); textarea.value = value; textarea.removeAttribute('data-initial-value'); From 276030377216a4b593085df10198bc7a4260464a Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 23 Sep 2023 12:07:45 +0200 Subject: [PATCH 12/12] tweak formatting --- templates/repo/editor/edit.tmpl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/templates/repo/editor/edit.tmpl b/templates/repo/editor/edit.tmpl index 759e46ce7848e..abe2e345fcd82 100644 --- a/templates/repo/editor/edit.tmpl +++ b/templates/repo/editor/edit.tmpl @@ -34,12 +34,13 @@ {{end}}
- + data-line-wrap-extensions="{{.LineWrapExtensions}}" + data-initial-value="{{JsonUtils.EncodeToString .FileContent}}">