From 08dd3f234f388ba5a99639a9a412d7fbeb494e02 Mon Sep 17 00:00:00 2001 From: Ramesh Mane <101566080+rameshmane7218@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:55:45 +0000 Subject: [PATCH] fix(nc-gui): csv import extension encoding issue and also add all supported charset --- .../nc-gui/extensions/data-exporter/index.vue | 2 +- packages/nocodb-sdk/src/lib/encoding.ts | 35 +++++++++++++++++++ packages/nocodb-sdk/src/lib/index.ts | 1 + .../jobs/data-export/data-export.processor.ts | 4 +-- .../jobs/jobs/export-import/export.service.ts | 4 +-- 5 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 packages/nocodb-sdk/src/lib/encoding.ts diff --git a/packages/nc-gui/extensions/data-exporter/index.vue b/packages/nc-gui/extensions/data-exporter/index.vue index 43607d6949..385dfd634b 100644 --- a/packages/nc-gui/extensions/data-exporter/index.vue +++ b/packages/nc-gui/extensions/data-exporter/index.vue @@ -282,7 +282,7 @@ eventBus.on(async (event, payload) => { onMounted(async () => { exportPayload.value = extension.value.kvStore.get('exportPayload') || {} exportPayload.value.delimiter = exportPayload.value.delimiter || ',' - exportPayload.value.encoding = exportPayload.value.encoding || 'utf8' + exportPayload.value.encoding = exportPayload.value.encoding || 'utf-8' deletedExports.value = extension.value.kvStore.get('deletedExports') || [] diff --git a/packages/nocodb-sdk/src/lib/encoding.ts b/packages/nocodb-sdk/src/lib/encoding.ts new file mode 100644 index 0000000000..3700b90f12 --- /dev/null +++ b/packages/nocodb-sdk/src/lib/encoding.ts @@ -0,0 +1,35 @@ +export const charsetOptions = [ + { label: 'Unicode (UTF-8)', value: 'utf-8' }, + { label: 'Arabic (ISO-8859-6)', value: 'iso-8859-6' }, + { label: 'Arabic (Windows-1256)', value: 'windows-1256' }, + { label: 'Baltic (ISO-8859-4)', value: 'iso-8859-4' }, + { label: 'Baltic (windows-1257)', value: 'windows-1257' }, + { label: 'Celtic (ISO-8859-14)', value: 'iso-8859-14' }, + { label: 'Central European (ISO-8859-2)', value: 'iso-8859-2' }, + { label: 'Central European (Windows-1250)', value: 'windows-1250' }, + { label: 'Chinese, Simplified (GBK)', value: 'gbk' }, + { label: 'Chinese (GB18030)', value: 'gb18030' }, + { label: 'Chinese Traditional (Big5)', value: 'big5' }, + { label: 'Cyrillic (KOI8-R)', value: 'koi8-r' }, + { label: 'Cyrillic (KOI8-U)', value: 'koi8-u' }, + { label: 'Cyrillic (ISO-8859-5)', value: 'iso-8859-5' }, + { label: 'Cyrillic (Windows-1251)', value: 'windows-1251' }, + { label: 'Cyrillic Mac OS (x-mac-cyrillic)', value: 'x-mac-cyrillic' }, + { label: 'Greek (ISO-8859-7)', value: 'iso-8859-7' }, + { label: 'Greek (Windows-1253)', value: 'windows-1253' }, + { label: 'Hebrew (ISO-8859-8)', value: 'iso-8859-8' }, + { label: 'Hebrew (Windows-1255)', value: 'windows-1255' }, + { label: 'Japanese (EUC-JP)', value: 'euc-jp' }, + { label: 'Japanese (ISO-2022-JP)', value: 'iso-2022-jp' }, + { label: 'Japanese (Shift-JIS)', value: 'shift-jis' }, + { label: 'Korean (EUC-KR)', value: 'euc-kr' }, + { label: 'Macintosh', value: 'macintosh' }, + { label: 'Nordic (ISO-8859-10)', value: 'iso-8859-10' }, + { label: 'South-Eastern European (ISO-8859-16)', value: 'iso-8859-16' }, + { label: 'Thai (Windows-874)', value: 'windows-874' }, + { label: 'Turkish (Windows-1254)', value: 'windows-1254' }, + { label: 'Vietnamese (Windows-1258)', value: 'windows-1258' }, + { label: 'Western European (ISO-8859-1)', value: 'iso-8859-1' }, + { label: 'Western European (Windows-1252)', value: 'windows-1252' }, + { label: 'Latin 3 (ISO-8859-3)', value: 'iso-8859-3' }, +]; diff --git a/packages/nocodb-sdk/src/lib/index.ts b/packages/nocodb-sdk/src/lib/index.ts index 656ea59fc2..3836330074 100644 --- a/packages/nocodb-sdk/src/lib/index.ts +++ b/packages/nocodb-sdk/src/lib/index.ts @@ -36,3 +36,4 @@ export * from '~/lib/filterHelpers'; export * from '~/lib/errorUtils'; export * from '~/lib/formBuilder'; export * from '~/lib/ai'; +export * from '~/lib/encoding'; diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts index c0abf1c38f..61b51104ea 100644 --- a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts @@ -59,7 +59,7 @@ export class DataExportProcessor { read() {}, }); - dataStream.setEncoding(options.encoding || 'utf8'); + dataStream.setEncoding('utf8'); let error = null; @@ -79,7 +79,7 @@ export class DataExportProcessor { viewId: view.id, ncSiteUrl: ncSiteUrl, delimiter: options?.delimiter, - encoding: options?.encoding || 'utf8', + encoding: options?.encoding || 'utf-8', }) .catch((e) => { this.logger.debug(e); diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts index dc59f6d9b3..b9301b3b63 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts @@ -538,7 +538,7 @@ export class ExportService { const hasLink = !dataExportMode && mmColumns.length > 0; - dataStream.setEncoding(param.encoding || 'utf8'); + dataStream.setEncoding('utf8'); const formatData = (data: any) => { for (const row of data) { @@ -641,7 +641,7 @@ export class ExportService { } if (hasLink) { - linkStream.setEncoding(param.encoding || 'utf8'); + linkStream.setEncoding('utf8'); let streamedHeaders = false;