From fb213eee0ec224c46025df33b4712912694bfa3d Mon Sep 17 00:00:00 2001 From: Mert E Date: Fri, 28 Jun 2024 15:46:49 +0300 Subject: [PATCH] feat: export as on background (#8890) * feat: csv export job * fix: duplicate job * feat: csv export job final * feat: data export extension POC * feat: data export final * fix: extensions & scroll --------- Co-authored-by: Raju Udava <86527202+dstala@users.noreply.github.com> --- .../nc-gui/components/dlg/AirtableImport.vue | 10 +- .../smartsheet/grid/PaginationV2.vue | 2 +- .../components/smartsheet/grid/index.vue | 2 +- packages/nc-gui/composables/useJobs.ts | 63 ++++++ .../nc-gui/extensions/data-exporter/icon.png | Bin 0 -> 37421 bytes .../nc-gui/extensions/data-exporter/index.vue | 180 ++++++++++++++++++ .../extensions/data-exporter/manifest.json | 11 ++ .../attachments-secure.controller.ts | 21 +- .../src/controllers/attachments.controller.ts | 44 ++++- .../controllers/jobs-meta.controller.spec.ts | 21 ++ .../src/controllers/jobs-meta.controller.ts | 28 +++ packages/nocodb/src/helpers/dataHelpers.ts | 7 + packages/nocodb/src/interface/Jobs.ts | 73 ++++++- packages/nocodb/src/meta/meta.service.ts | 11 ++ .../meta/migrations/XcMigrationSourcev2.ts | 4 + .../src/meta/migrations/v2/nc_053_jobs.ts | 30 +++ packages/nocodb/src/models/Job.ts | 136 +++++++++++++ packages/nocodb/src/models/PresignedUrl.ts | 17 +- packages/nocodb/src/models/index.ts | 1 + .../jobs/fallback/fallback-queue.service.ts | 12 +- .../jobs/fallback/jobs-event.service.ts | 54 ------ .../src/modules/jobs/fallback/jobs.service.ts | 42 ++-- .../src/modules/jobs/jobs-event.service.ts | 108 +++++++++++ .../modules/jobs/jobs-service.interface.ts | 1 - .../src/modules/jobs/jobs.controller.ts | 107 +++++------ .../nocodb/src/modules/jobs/jobs.module.ts | 12 +- .../jobs/at-import/at-import.processor.ts | 6 +- .../data-export/data-export.controller.ts | 58 ++++++ .../jobs/data-export/data-export.processor.ts | 132 +++++++++++++ .../export-import/duplicate.controller.ts | 9 +- .../jobs/export-import/duplicate.processor.ts | 20 +- .../jobs/jobs/export-import/export.service.ts | 164 +++++++++++----- .../source-create/source-create.controller.ts | 1 + .../source-create/source-create.processor.ts | 2 - .../source-delete/source-delete.controller.ts | 1 + .../source-delete/source-delete.processor.ts | 2 - .../modules/jobs/redis/jobs-event.service.ts | 53 ------ .../src/modules/jobs/redis/jobs.service.ts | 43 ++--- packages/nocodb/src/modules/noco.module.ts | 4 + packages/nocodb/src/plugins/s3/S3.ts | 5 +- packages/nocodb/src/schema/swagger-v2.json | 90 +++++++++ packages/nocodb/src/schema/swagger.json | 74 ++++++- .../src/services/jobs-meta.service.spec.ts | 19 ++ .../nocodb/src/services/jobs-meta.service.ts | 83 ++++++++ packages/nocodb/src/utils/acl.ts | 4 + packages/nocodb/src/utils/globals.ts | 3 + 46 files changed, 1455 insertions(+), 315 deletions(-) create mode 100644 packages/nc-gui/composables/useJobs.ts create mode 100644 packages/nc-gui/extensions/data-exporter/icon.png create mode 100644 packages/nc-gui/extensions/data-exporter/index.vue create mode 100644 packages/nc-gui/extensions/data-exporter/manifest.json create mode 100644 packages/nocodb/src/controllers/jobs-meta.controller.spec.ts create mode 100644 packages/nocodb/src/controllers/jobs-meta.controller.ts create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_053_jobs.ts create mode 100644 packages/nocodb/src/models/Job.ts delete mode 100644 packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts create mode 100644 packages/nocodb/src/modules/jobs/jobs-event.service.ts create mode 100644 packages/nocodb/src/modules/jobs/jobs/data-export/data-export.controller.ts create mode 100644 packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts delete mode 100644 packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts create mode 100644 packages/nocodb/src/services/jobs-meta.service.spec.ts create mode 100644 packages/nocodb/src/services/jobs-meta.service.ts diff --git a/packages/nc-gui/components/dlg/AirtableImport.vue b/packages/nc-gui/components/dlg/AirtableImport.vue index d33078bf37..aadfd13697 100644 --- a/packages/nc-gui/components/dlg/AirtableImport.vue +++ b/packages/nc-gui/components/dlg/AirtableImport.vue @@ -22,6 +22,8 @@ const { refreshCommandPalette } = useCommandPalette() const { loadTables } = baseStore +const { getJobsForBase, loadJobsForBase } = useJobs() + const showGoToDashboardButton = ref(false) const step = ref(1) @@ -141,7 +143,13 @@ async function listenForUpdates(id?: string) { listeningForUpdates.value = true - const job = id ? { id } : await $api.jobs.status({ syncId: syncSource.value.id }) + await loadJobsForBase(baseId) + + const jobs = await getJobsForBase(baseId) + + const job = id + ? { id } + : jobs.find((j) => j.base_id === baseId && j.status !== JobStatus.COMPLETED && j.status !== JobStatus.FAILED) if (!job) { listeningForUpdates.value = false diff --git a/packages/nc-gui/components/smartsheet/grid/PaginationV2.vue b/packages/nc-gui/components/smartsheet/grid/PaginationV2.vue index c42e7ccf8e..88cd79868b 100644 --- a/packages/nc-gui/components/smartsheet/grid/PaginationV2.vue +++ b/packages/nc-gui/components/smartsheet/grid/PaginationV2.vue @@ -256,7 +256,7 @@ const renderAltOrOptlKey = () => {
‎
-
+
{
{ + const baseJobs = ref>({}) + return { baseJobs } +}) + +interface JobType { + id: string + job: string + status: string + result: Record + fk_user_id: string + fk_workspace_id: string + base_id: string + created_at: Date + updated_at: Date +} + +export const useJobs = createSharedComposable(() => { + const { baseJobs } = jobsState() + + const { $api } = useNuxtApp() + + const { base } = storeToRefs(useBase()) + + const activeBaseJobs = computed(() => { + if (!base.value || !base.value.id) { + return null + } + return baseJobs.value[base.value.id] + }) + + const jobList = computed(() => { + return activeBaseJobs.value || [] + }) + + const getJobsForBase = (baseId: string) => { + return baseJobs.value[baseId] || [] + } + + const loadJobsForBase = async (baseId?: string) => { + if (!baseId) { + baseId = base.value.id + + if (!baseId) { + return + } + } + + const jobs: JobType[] = await $api.jobs.list(baseId, {}) + + if (baseJobs.value[baseId]) { + baseJobs.value[baseId] = jobs || baseJobs.value[baseId] + } else { + baseJobs.value[baseId] = jobs || [] + } + } + + return { + jobList, + loadJobsForBase, + getJobsForBase, + } +}) diff --git a/packages/nc-gui/extensions/data-exporter/icon.png b/packages/nc-gui/extensions/data-exporter/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..112d158118b7c4dbae0476eb7d3440715887b939 GIT binary patch literal 37421 zcmeFaXH-;I7dL)@fQbd3#EPO=C_zAwA{{0%iogI0Dk4%MU8G3wj4?{A=nTD%Q9u+V z0)ilAjEQEbqo@c-QIRfPI?TK8oqI>$FaHnkxBpt6XRRlC=G=Yu{_T3oy`JB1X1HSM z`lSe=6-N7hu|Vis_~WS(qXeDuWR1 zcZ6o4i`I)!0F3Q-Ku9|gp|w7#We0TO2a)3@hQA;#_CIDtHUmZ?qhEeH7~C@07Irl) z=+_QzL!o@fgAI*YLjQiCKc81lcpd+pTvndr?r)_U`?}U9bU%-`Uupc@eM9oi>zSr) zyY{XTdH>|Qb+xu@l?uLh5VwghY8f?;_qq9UY4f+2!dDkK+r%rd%kl#oN}|^Nv9q|G zKIvOO6;k1%R;W5@pJ7lxo;qIMFeL@7eQjS6_=><+1im8h|B8SzWBp3RdKP#6x}{%e z&m{@DY^ey@OeLk@u`2)c1T(Xlx7n50VyCNGR^Mq<%QvpBVWf^ej#3J1sp_#=4wN)S zb?AdsMp~bft3}{VcD-72x>?7WzJWIxqt;?--ld)2TM8fsBrMAP>Nbrux?Xl_TvoZh zyWQOT=!ML3ZMHmp!sa_5)GN4z@g}2dF;@4g%v5%4*z>+O*GkIAPiMJbIFD%O|A)~2 zvnmCHAH;{lB9ehdYRdC+ef9>rq~tR(c${-_Ql>dID_kSfI)iTNK3mmRTj>z(j%e8{ z;YYvftm|vLMy}cK%O3VL3bA>fn$YE61CJra17lt11v-oiWK6wkv`Qkf-FNgQil7!q zsg$Xz8@4(VLC-t1lrqZ|<$Aayr@8ES0eEWqUW8F|qFPC&`{8W=MXt-F&Sau6YS{x1 zHOYCrN{zP^oqr$O-DVVGMX&N+4NuSJ2riI$L$ET|)0J zb2@oIbukPx~*G)@ix<{(A<#0aTE-$h4J(N7whbuq`m7Rfsj)%`BWMrsCC3orFm4=y7 zUa=Cpr?KhSvsAygYy1$ac8nuJ`%dlyz4B7|f}UKOm~5TW5o~H&cCS>aT;JTd%cRo6 zeJ$kR0-S5!E^w)H*K+4Bd9D_f*hK|ifboNw;#BT*YIkao=z1ZPg<GuV zb;($pyLN(XNQF8^D)DDPFg9yuzhC`Xn_BQb6EL&mAM&lrcPz{V)9vqL)g)@{b)Z-M zcimFSv)&*oAB6P5$pUh3jw#Y1zydq&nF^O*eSV~X;Iep zOU?K`=gQU%E(=T)LRZqjjC`Gg+2i!aMFqB`Y#(ke$kxj$IsBH^)SnHBg6PU+712t)9OXpFo7L(-*j9R_ge&Pb+-7KrN&S$D;SwCG_s7 z50z<>9y=<7-$)=Ie?dg6jtw|d#mG8#t;_ipMvA%A^wmxFuzJt230q+N*^VT1@H&Qs z_WKNDc32+H_*qfuI%%z-mmf_1jGjuMkw%bt^^XHfqU18ewb=4J>0W|*8fUEPw1(ff zEJwk!&Ogx751orPH*Ws)R;@0|{JATx?Uw5y3Yjx)-R1UWkJg}(SoxdZp-@Zl$`3vC z#j{5B#khv@Uc`nZy_{L-uB$5VScf))Um7_Fj|dr+Ze_dk`2J9t5<9i4u;Wy=9CFhU zKol{#-Mh29_q>maqjMW@m_{6v0__wRqN{R0qE5QLkh5E# zj?5VfG0|L%kn;CEvW4erS{_BYEk!L)U^3g=Zz3Bnstj}~$+dNzSUqM24V~``gkT;VovrDzPi94LUY{xoP5p*Boq<`z&wYMLLv4#Lt_hBF;vL zxLg_2N^vnRrdWsVu)o4aD2iWim{C>9`*|RF`i}tG`_k z`R_v@b9w$7#9F?6_wK^6zekm1W)wXz;aNrErfhHZ9$WtkIdtnhFzvFhxv00R+%0_b zrbQuagujlKsYvzgqp2i8S~9qi3#D`}%hI=ZV!&c-fQOFwvkVJ;t8-TKhL&jmG6Mmm zTvz?XCOzZcVt#hlR(v9g)pQ|56mIKsG@&hkSZdohoU^0f8%=rMwU!5-*H-q%)FDbf=$d*)rF{NhBh|#NW4sC0%73Wt zE)NRl{0~k1slO6YN9WGIm2NOe-thjd0Vb<|Ro2X7c&JKTkY*=^#T@6lCJO1q>v2{RUD^R#FC311ir=Cvn zaeDbv9+!;doAZaPtKO9W=J`+oPzx+}^woIS&1T*_b`rPTUiS#Ond8<5(+x1h``81N zcx+owgVCQ$F%P0?*BZ+$BdoE^)=`kgwmi2aN<*&ch#2Ap zGDa|)t(-O8DV`^V@l=-g#M%_U_Ju?_)VnuQ5P{hRl$39g78)Iq#E6`jZ1FdnOk+c%UheO7opFRWi;V?v~0g!hOAH zPpRfW1J_RnaXcZeJPo^OZ_HqrErvr@#L+t1QxzLf!_5>&y)qEN-rtY>H`;P+;2g#R zE&fFEotm-_t{VhtcT3fhQ0MEo0Qnupj7+0h9tU}olqaeWXU<*>Octaib$%@jZpMX2GB@QCfL_OmZ&B954^DzKSF2Y$=AQNq*oZ$`h0*spIaG7qGOALGJ71B7 z9Lj>3!XpaR8}FEz+jr>Z$6-TQr-Dt*w5FqK!9~aRFY9c+bj2>MD~>n86_dvDd}$ZpK0nN?av9tT+(QwFJ^Hv7ll({J3ewqH(e>gLt%9abdw# z9V-D>A?j*bgy#2`ek@*_C6`f*L$54smp(dbQioUv0nGXT6>5%MLWq&MQ-HJAdEtV+DfOd7k!3DQJ&Q2(c-8aCEaY3EI-y+(q`a^CA9n^&VHu1+)!iwEWJB83wV}xGhcNvJ|{k>b1 zYF|&!0W$YaP|=IrT$@t|k`+pDO>-PKRlCPjWoZq(J79^F4}f`}m@<^%O~sM-?VOkb z0UUv|+lOAYjh$1QapvhmFQ;#gVR4}`RUlDW0PP<)c@uRZ1JV_=(8D3S_Z}{96h+in z;D5Y$I^QbwTGosy@noJFxBXk*Z31tp1FQjGIET;M%Y>xEeBcn(^GC(4Cw)PgYM|&{RUWvCnLig= z%r4#s^{)pxFRK)c)fu(B;pWUV&Yan}jW>K#TD^e={=okHF+b-vMeo00^1U#J|M=X7 zT{u^_U)7H7`4Ocy5YUFzIN!f<47I={{YKbirRa zQa%MghxE-k4h^D^>T92oSuW*-Ox*(0M4Y z4`qg=;PpB3Jt0~(aZD9m1*Xyu;RrzheLi|t4y8+h0h=|eP5dV1U4Z{WdAH0G&Hn)e z^p-s7NOp?Htp(AgdJ-(wtD?;?ek88u)M0!|^U;FkVionvVOyWpZYkl4$KnyjlL)V+ zG_1le1gnEKvdXkxRSGhr^sO$i?#a-WCol|Q}b5A8*6qt2{vC=EaII+28F-?-Lx%0-Z({aF|Mjy25- zPmFFOPZ^;5GZy67@(Q?_`K&Ut5Oz}svfCz8lWum1;-d`#cV4GP-jzI@xnMI;G_wvu zZ_JybUHJsZe%d$Q1hy^%c+3DE4YiQ7JSJp1W1l5#5MZXkc<&#z9^)Q>R_>fs*?@`| zpq=kGYUJWmtyHO5dm9~vC_1r}-!-)C5K_c2_~re%)z~nHpTpCaat^~BRuAyQ8kG2l zzU*zmBG~-kYDw%$wMSj z8ESwch?>2TX^V}QaAuT$lFnYw<65lTBF>WRH4V>N%CUgi=Pn1B?cyox_ZjMHW+uot za@F~RuM(66b(H=bDk@X7h{ zMzQXVByH!7vQnub00jN<ZiGdIaY2@$d>(H+V zd_~|Z0$&mMiojO{z9R4yfv*UBMc^v}{|^XguIZR{8T-df2M(YDhz0)BclV;{GYQgS z9p1|jD!ag2cXS^4*uhwSEd69z+Zs6e;BBQL*66woCSHp_`wqOjxhf2fzTT5W?!-EM zeD1_nXTQFs2V1hc%YddcD-}1qM4=x)>Ih*ijYMWNhTv{nG5`O0w_YPh$PVA}Xso ztwj2we!9eKS0J60${X#lT`$46$`vR7{#U@hT zDEiU-Zk0;2vJ*yvMa8JyJTGih;Dj9$qiUWO8udnp&fw zipM~D{u_dHzPMg&_(ul&KVv{+#F4BXlG`7Rt{Ul%Io zZN*@eDfz8qZBC1ZblVufg847On)>Vede%PH1p$SCJ+{Qh+Ww1Y<`@I|*TP*dW4-%` zUdHHXSGSVP86hLNNgxb6ZdjvS-g{W)B3DIR4=jKdk8dKvmSumX4&$t(X?pM*5JPb@ zJ`~+*b`--zj?o2Mroje0-cFQ6nHP+Vb%l8VJo{m94r`PRmn4lUXG)#w=(zVBB)H(L z$&|X2Af;i)xOBNi2r+pZHi%YS>Y+JPy_$Vm7L>Z5iun7f-2~#d_GOG*>C?XB+#s8v0Yod&01zM3~kHBhb@)O|5C9PXadh4`Ogepj(u0 zTO9k`7SQR$w>&mbLvEwd;bSSwMWr$wVZt4-n}fB^UVz!YVmlA!9fT=&mbO*NS!-%E z7h=|8OX!dFkM87tmEmpdGeO`b%w+`5i4V@q(Lt@eFl9@4FzcT9{}rbEzatoVdKU{* zUFRC44EKJ7JtkqoWF)HU1{mNz5t#cZKPGxp6TfgD2N7aQ%~N4PvOi_|2Z=_gTLT!~ob7>tAZ}^$u&--=Zd-k`T>6eRvf(St&^l9()u7M2WS3B+lg+`Ks zVkYBFnWptFrM^p`+sWt|}thVPd-d8FN((h13+?dtfpY4}#-cjgDf{<4Pu*Dm$Q8p6qQ})MK+gu(Mhr zbR0A%fM|R2gq~7$6$Hf8O1V!6_agtxL{HmJ@a7gXEGBDT`i`9;0zrwl7YiqN&4Koi z@4B8zf?978l?83G`Js;Z1Km>B8^ACeVB^g#``Kf5lR;9h_3>qJHHiSe1-q z#^|XqIP@^(4Msuj@d~rSz77zmXz4`wqDh5}q&q!Ib?SKxx(lTfLnEEO$nhLMLX44v z(UY=G_DLs?xWjbW1QVTo+G$SzLh@;6LK%xIyNk@x8n&}NCc+*Ut(~w@) zBNG+~!u6gYdV_c96ibiqFfI0i49Fo-Q~rL*6h0kRO;N0fY{~lN@?~j_2z_yhea7Ic z9P-G;Cl}(PM!2??$_ZeOR9P;uHVuiDs5X7gv*xM9h!=`4y%>lc7gtg9J z23ymXhbxqZodJZzNrYAgU@__rL)`>yuy!3`K-6;<^p_@d=_aZ7hJyn9!V6Ke9=?(m z0>7CAu@esy0w$hp9l|CI)IA9kwh+iUY#(gCrjW>Z69xqj&3y}x4O)&$=#MikvW#nS zpc1keLs=GmQfGwN1tCVRZ#E%TEMc=O|JM(;moDD!;s`)&u>6@v12sC=1TkhLjRN%m z4TtB1gbvytThbLCC^VNfuR*MvB%=O04-86&R0Ce%{zs;GO#ngx;O7IL)vR9lk5yxZ76?}6(pDZH_$?9msAzViVgIMZgZgP~Ba zJJWs?n0`vcX4K|1zi+Ir0kQ$^tk_{N-uzQ=tT*HvJ;4$X^$(KxOWVh6tGoL|b$?xk zI<*KPnJHi9`YS{#L~ULjce`3*WTqlpLH0sIc8bzED3m|;JAJeOP8#+Rlq>%9r`xQR zEvYrp#tP-g--!Oux{Y_}`NytFyn#Zw^C@9YoQs>vb)a|}o8AVp(@7zNFt8I-<~w%) zX#%ERcg%%slTNM!dn_Vq3lwUb&B%SQ3p44bn-F%Gg11}${#V^~hj1aX`ahE zZ1ie}sn;|W($6M-0fvemb$??2UCOSsg^Q`iB+80kxX>-4>_II~!$VG;j_kG$aO8tT zVSzmTzR2?1xAJ(S?b?nN$=0)(PT^qCWc7uLn$zwu^1~Ih{%Tl*IOI}^6DZb}0jmOs=X} z=B*F5&-5Q+zdZxeZ6bUSr8f(u&_f9S=@lRp(%JkmRE?~XnKc9yv>TZJELL~vbbwO* ziMkbyyx}DSBK4G!7$rWy)*@E2hI@f&viL$0DRmpLwTSGd5}-kpt;lB(P+oH^UU!V9 z2)958%c8w6^bX<|%C;0j*%DQw+t?Zg3Axtv*yCa-@h6h&e`4vmcJw#X%}yslulxli ztLuS>Z9(saF4jsAU3@b^My6r+6qfG-y$Xd9Rh95TB=03aSc8qXLr9%x^?JrEElaCaZSc=REK51BqXfA2$dG zI}n1j8DBBhF?z93+=NU=+91|e<_*Wu zDg*T)5d|+L(Bp&!8pK2JF8BbUiX}$cb-O>TOBP^V0*Q!5(zSclCnYaJ=dI|ElA(K| z@^iMQ@h+kOAOT-2=-s4=$@a%=Y^8@J$k=GK3esMYCKI5A`w)&Ylg#Oq2+W1_Gk7jU zJ4~wN(|V|qNBX+{yQPS35mh0&)!Kc{>Fpb*VUS@zSrqQC`n(#Uogf7Zd|jd=K7GV@ zN(6?Id~r5?v6eq*Na(Z31m7~Dd(cO>O(N!c!tj}CLq`*qLn3(Xu?*ebObl-EXsuw+ zYq76HdkRn*NUQ%?>}m+xD89oRi=k>FctCr_r0wUNR0^K2K@$oDG!%NQnTqCBg_H#U zCm(?V36Ac|Cx>7WAssFHT9B5$m6$EKOtHhXJ4&kg6Tr~RyEF%@6W0Q$jyntR{SwYf z!VyqIH|&>S)v>kXGE_!l*6H@-DI=vSkpP(Y76gz_{NrbX^>Og_;u~f!Ln*l}S2kv6-gjDZe^cv)qkHzhZW`a}c*nQ;SIMEdGJ_0_ znX1LSxVg%vvdXzA92(Z6_1xoQcQvFsxK}_{Z9)sQ-J?@c3FYrK=-&FU=|FD7q0apa zBVp`^i^E-~8(QF|>lOkrO$qaPshP(eruq@y7I*Ye;wh5Q<3k?49Tn^)kvbvsPbZPX zUQ$h0Uf4(JH!nys3@c%$&xI(`k%LNjgl1z1V)%R(Cauq-{MN_K|R$~Z4OD8MQrEu0$1{ls>filRnN2-zUz z52P9``-T^Hh4c*2hr?lwLWL_Icw-$5n{H+6*k^=)$oCZB7l()%Z5Wj}xM0ra&L_gW zKak?~sqDNqKdQ$z43rM|dlRC<1uPyN|GDuPnc+ykvDYyt1z0yp1FZdhAu8$3__?ky zE^OGyoaK#14q5XHP4p7(T-Wd-tTrXc)W@REJ`|{TE_dj!ET| z*pjWZQ6(_945B7Oz3K*$Y9S3#W{dNVKqP}tVHdQ4(^E{&&W4PBMz@2XX z!gHHOg5LY7*FsFwc7hy7vioLNTEN_YZR&E}$c@ieRP2j#8OO&{FY>Z$)t6B47lg1! zBC;80` zFc^4VI~0D`Yh&aUIs6?roDR*tNF{Bougvpb=`7SsGKkWv@5z079B{HL5atviNtnAT zv!ila&vhIb4!rvf=VSPOA)Lrat#PTm*rjl$UDcSIBl3p}aJ;tUeYLLsXijL`h&xb+ z+h@Ec`l>mk{EdBysJ+8d+7M}Jot<&1v1T*BwC$Sr1tM!nBGD%Tjb|3hJi7ug%zMer z9VBXK#J;0id_1TvwECDIs?{YQFl2g}y7oE-42-Sx6i_ZCag@l3iZkmO4>F1bIlX%iy?dc|IltHWCza9mvWPA(OGJg&Qh1G#bF*6=PzmaPb!c}tO4KJM%KBST zeEOMT|EsvCDS{56WF$>oudZa~`fgD>u*LcbN(|)>e}2KKvl6IhW}(ER{N9#7Uxfd4 zWyP?XFzW>wwUjrCFvhyIcg@Go(i4^^3ptFDnoiv7Y-l#!zoh@wwX??_7*}Q2bUMV$BbuuHrkxJu@hY(*K)KZHJu#q zJ3LJX3j0Wf|G}fKJdSI6FlQj4UR8wkmgJJ7c$ZzSHTCkNe0-8%Cx>LQSEeVcTNT`8 znW=z}8>tBYq%t-2D2tN(*C&_J&JbI|h`FK8Ja71JtKrZtVb&u);XLCLc9u`~4&`Vl zW;lw{TxAH(nXR4;G1`(j8o5V~FQsMksjSVdi;*45x3SW1T7}HXZ6iv|kK6jF;aOKM z=N1aH7<~Su4W&cud9F3jx1S1^&FcAg93Ok<=-4UONT4O8I(rW|8&=MZNWLBT^X%|4FZ-DvEYSj)3nUUIod zkw0fK5S7ORTvTL2QZk%Jy(lyG;TY?`ichm~h8u)>Wl1F;^*^*JW7?@r+xNB{Es|6x z^H761+qM7oe=}KV_B6>~Aox|+O3*^U6LgdhRDhqw_!Th0q2yJ5D34rWa>M{{#=W;; z{jV&$lZD)_kwo+@o;*)lcozC`Yl=N9kG1#!LZlM45g^s&8+!Q4eKuUk!%;ICoyr%Sf zuh-yjZsCtm=r77OSH^VM=i5eR?Zn^5!ux4a_f%G<&Gp}x-)kx`L7d0IsZ+V5>H&j; zC1M@!OPL9zDm&RCmNnk5JRf`K+Ne5+GS~8d=!^BWsvg0;>Io|2-`h+#l2bB~&B$#{ zP|^_L)R7{M4Az@{gs3W6`QIu^35g=>k#*|PMZ-4z@H28WFDXJ&+8GU<`{j9F8$xU= zTCF#5Y{P#Pz<*+0my-(#SUlkrWD75^Ytg(&ebOEu7^_Lr=&1}{sJ6T$;4j1*5Bukp ze&f^sxc0wVt)wihjJ=zQHyn`gGVI%JW8I`~4) z%JPz&X)6nOc~y(nM|u{GebCQO+hEqM;@5wFa72wp1aZU|-ZL}$&qBapQ6W`;%8$>~ z^D6Y^^X_f>zV{VreVd8Vv>d;KC)(|w9xkt5#X3l;6pk+T^{4c|9yxL$f`|Dvxv&~+7Rhj(OSCErz@OTF0-iHKit(-E-1NcuidbY zb&afCS+%gb^V7F2xsrNfX{QevE|@y040^*0yljvVC3WF{kKL`j#at@->7<{7D1gY*=$Mb_!_PFwZ)>x1Rq z-6dSa=WpQyzfAaC-f^PVNUa>s!*f>AZj$c8|5YW>+R2bhTMRV8{Hm>NEZCCbP5J~< zGA2rdELuCUVGld7PJ2{*u+(G$LrLC{!TlfAC377g>zE2u@L8MBeRQ+D^va5=%*rn5 zGu^ayGJK%)pPR3(teibv7T|PUk*PuAN6S8tzfU&fHpH?LVSj#gGeS*Ih;3-{e7rVA zu#DXNM!_!kB@!(3k3Bv7#%+189cdt)4z*_et^LJsHC{bTQ~g*VM zFFu*{hznsKGvGDNgH2BDsdN34t81tNZG5mKuJ~DwCqW)Mn?)PiOb!;cv`tPn$Z54Y zSuFgyutRDN975Tww{rI8IU8#3Mr4$HF~Uc)@+y6WbJMhysX*ILBsGcKK|R+Dy%nY+ z7rwWoO!dM?qVT5acE3{2^|H6$v{>8I4MZpH%;*t|z*xQaRo+WKm#m#hakDW=|HT)$ z&aZqz8{(@eoy|4tpDpOQ4)0sHsRnlOI$i)B1z%pVfmhI%xe4o1w41>(LU>Npe|I8K))QBYt zhFbEs@27@pesMKz{x~8TN*wXR*RfiMU5*dAHP3Z+qzHzl$P*GW4PM5U7Z0wlYF)+Q z*Y*<`l2U023ASRD9jiF}*fx=nFP*l+dFo7**HSmWDrcW&xEw!j*{ri@YDo#xCLE0x zxqs5--P45F3Nfwu=)ECA#B@b@S=}p|3x_-6ZYB%0kjT*j_2Q3uD~3&0JzmApCq*v0 z)x9Anz_R%vx9|yVkwdP<)p{<9=XwB`QDjJ`*er&L=qT+NZs9sxQa*RfpNO!mUvF>K zKGQh`;l-7-7tZ)5b6>0GLN2^qy}*wE6(4nSJ|xtnM@Xs)PiT-#Ox#_fdY~z;;(e6k zQYNX=UZbegr!w_QVN*KuOUUC9#GHy9YqfsRdwZ-hy3%8LXZA6YNX7f}vGYfp?+<^< zLXl(@g+d>TSqua?Pc4R-3h1sT$NE<-^!dL^2-`GoVu{SN`BNUxY|k*WeLo@@yoC8&R3LDchsagPWfKC~L0ckKYlK)!khXHowUdMgL4_1hxIy zy+N+|$mFG{`u{6sO<#O*^2e|zRmUOek-@^DyFxZbH*|OD)Idz4pysiSgrBpP-~ME( z}8aa89%zOrjCJ+-LubZ(e#g%W{pT+|W^?oRU;9@Xv9`d-T&3iz}`BjoPq0;YHx z?0C-klX%oz>I1mXsD|uZo&Z&x zM3tHrWz4?s+*E0xCanCDOcKh{{lO~t?hek52^FKP-K760_YKwT=lRd)Rb3)gnrko9 zeI>qRc&>OSt*?terG&b4zT(3t!@Go)`MZ-oU6F}L?d$@F!!qP(Nm~i-Xxql`Q)4eX z4_6wf3M(`C&hmKj?DD`J{?lUqLf+)i#|Oqc#%>te9~fGg-3Gw}@a zTP#uAZPLbqMeau`uPpuVtpVp_IY}YiEYI7xjq6?0uKVk9W-FP~JK3AebGTW9?c59~ zu*rRQD6ofbS{BELJux`(Wr1z4n^^zPtZ&2aD9@$LCVz7MV`Qu!SKqlln)^jTUp?H_ zyXVG-bftweiZr_qB+9eRE9GYH7kG5Qb_n+62VPs~Un2Kuw!nicnJDOXnuKoAay+Qd ze^;!n*rK^3$|FNOZP7xXaeDS=n9hSlV&Ynv&dQj}@nMruHESpyq{mVWZpnps27csz zq961_ie$>=lo?w3nl!5~?yy9Pq%CspCVSs>a)e$jn2g_hn;Vz>32fhrDll>83IUrN zN7DH(zXQz!pgKv?En6%GQhZ-c&yLkEsq$XV%pl#Ec6eSh^UdMF7;ZW&>!b+jn8F=K zea9>V^`>QEU5CHI55Lgtrb?1-4b-$R;XYYKF&H4c2VUSDHyC^UkFfJ_B#7O?=k7+o zjbH7bcWUM~&E8c+e6fc^wn|qW4*VDuQP8m-aid8@?Z(WF?#e%WVgP$I8Dt5=G<-2i zT@D^AIWKR6bgfDEnV^}w-YR_Z)_~Q>pYLO!^UyygHLX(-@G`%71OAF25cj{6OmB*} zeIJw}*x5{)R_|T|s+>lw?OeOZQra0Z*D-5i^Rr)E1{ZE<73}Qf50Njt3eJ8RHKf?o=%N1xJW3?b1W?xV%C7UD zoSPCPm0>T442pTZUIo>W9@nv_o6bM7tw~) z-3b3mfAZdFXnWWsHy(!bWy~H+dofny+)x>lB>1@)p57zo^56C?kQ)Y)gUZSBIuW?>t+d#g$LthH7;!7ju9&HQrCFlm_;xBThP?m2yetKpl^lVVIQuQPg8r6R5c zw`1nXLw|~r^#je)?nCTTvdv$fYpj;d-?yvnTCGksoTczdtW90`v#TIMRtTzt&nG5j z)^hBY0CvkK=oBaQSoSivnf4V2;SdtNL1Nsc{ZCAHRDe`Gm|-beH*Hv-1;Ui;D? zK7nJZ#isKFBQBgb!f=I;VQB`pE)0JmM_XPo&}t5ME%3Rgh%}EAjzLROF5hllwJk{x zPLQZ%3TOG-=p|qBGBJJ%o#e(D%fAnUxT_qljD4=csL@*EjLRj`+39e7F6(z+V~+*eJ{N}z6qfLUJ>Ha?;cjxU zrSGE}7+XZbM$Nk3%r3K`uLhH#xsz*$M1^H*q?-n*@d|LeqmfTRJpPJQ%J9DKjh=AG zNwU_tFKy=UuId!}Be+SU>`2;AH1^s;nNwYvPy5 zvwJsJ*&t3FIa`C1!{*ZYb6p{DRh(i)=1twSQVe0)ntF@}GKP*ET%%}oH{?)3?>9h# zUvpOV7WK%0%lLJJS*33w7@oH7ITm{?l!ve$86K`mWimj?hhqRr?5hrVTbInpeW-3d z8V*#+T_H-8%>&KmvD;3gMqeEE(-!ah?S(&#(_^NDCF76rhlN zQKICetsm~Ne%QVSsgd(f#8?+H?s%J-89-uGBX=(*qJ^J=)~IH>a2r?6h+wP8n_Ylm zdutEOWlb0_)6+HA>~5J>k~*A|CzC$Zaz9i}e4~jm+)=M1G5jblLETZc(>mZf9JFWT z8xZ}bNje72!C?i1%RyQzLJbt1HA<$%gVZ%UiiJ@G$yU*?!G{VW4OPORRO&S$8JJ4* zJK)>*$tWA{AI!!Ry!P6rnW>a1YI(zT=LtQcOy#6>*P8Eo6Q3HGrHD3@&#&F+7hd4( z>iY(OrEzx?1SaAw56f%-!&L;0&yyR19G&{eQ@+ig*etj<)k*G8<5y(@Z3_Y`pcB4` zgE!jHE;XIf*;h^KWf%!f`%FR?E^r&1 z-K|4soP%3o(Ik!9-1p{Y0Pq$4dKmo;F(Io~JkqYF z&m1M%rJ0RA_h9%%vEdTcd2;APlBz-TbUwo}1GJIoM2P1Qs_;1!@yje)PEd z4~XgtA=RJB-pbR~OBVaNxez;%D!F=+NXNZppV=|5W$+DILgr46qILYqN>#1dBXCRaSkmQ$EnhtJe1&eX)|C#~yO;Q8_y5kjd%yx7@^_&}o>N&$^cJShdYElX7i;I+; zhWf|@zRfj^({PKJ5<~br#Q?5o5}}CmE#aLLDR?L{GBbcQ$RRh}CKxxa80~6HN)1j1 zX?GIf_O3OF$CAw&1-KZgMkeEh&P!{htQL$_JpKa;#5oQs%8l)sm#1IUkPH?<$wa!M z9bO?SWA)~$ZV>uV^-3J2e6S152N+Qa)V5Z*b?QswS``!=M zC?@Buqz8iV$_UMp#puy{5c%5TW8pRtlf3W7Yzok3_a5)~$WNGwU=;~E9yyUj@o)-G`AnD^Z@>AX61HOTS>Xv z!lfR53B|OjwNE@*s;YIW6mF&OCUNlcnU`b7r?HmY~t zye2qSW*m-@x#4Ie2RvLMoyX;G7R7gF1$+Dz5Fp>~Ly2B~J+d3T&EwPHK8?e30!jyM zD2k4a-*$TT9gvVA3&gA%_Xg8la1XT>P*d(BXWLj``eW%t*-cdt!@-y&u{{n;E!Ox2 zc=Fb=!)>h0ZA;D#t_0cSh^09#=P#dA1s@Kv-!Z_|sLKdTrxs_*)9Zqy#;<_&DtnT% zCDrR1l=f#iIlv1&{JM(9-fvt$cD^!rOMp+Q? z7*#sd<;*)o(CX|<{hYg*17z=7%w3AR$`8PFdSpU2@Ti}6dhE$|cLgo?$`OHJI_Ha6 z#J|@_p{Ew8GzS~1c%KG2$hwMFXrFp4FhWHcw8?ZOChN6_Yqd?%9!;%4HH4r#+xSIV zaU-mnw&?d_C%Ze1{arS2;C7B@>$JrJe<_FHWb6rF$j+*3c8PQzoz*#D{X}M!CjuSRg)HlswDC#h7#DJZ0){?!h z(;@708`&HQ04#z4jAHZIRc0>`%W=--<>ldW4kw&~;%5cgTNt@|SBI9{6Udkp= z=4m=y%x313y|&qDaVH-rCaFTf$lu7~qD4m3SDDaDvJ+hoY!9V+pls$5Z?9KQhW&2zudlSA$*|go{ zbYTjgQ#O(y-@j{n`$fv`R3?;}7{X84NJ56trK;-}Dck3!K_0J0U|mZ5T;w9EAThde z_(3LH9o}NS2(g}jwVay&;SPO;4!zkA>m@5IYQ`5yTbWJBL9O_lv>h&xoE_cmM~BtU z-g+pWS*wU8o^rqyEq(AytsyDUJ(WDgv3%dPnw_P?2*FAke zzwp$->iWEbK#fHC!|F;@hdMXDYXaw2h@c=*`bFOU{9fj^LMLnAx_YNaSS+d~T_Ae+ zi^R<@tXS9OrM0*hy57}c$%TKVRm&=S;1jTt(2h?HIuEZ|k~I`f z7IT?k8{hsUNF%-96Sf`M{--pS%I?WRgkVM!`j}puoeFR6>m@zJz|`AIfGN;W^<(KE zFS+n*tKOh5e5UtT9$i`_EG4XiObXr|1JouodFDf@I{Md-r7o@oPD#ModcuZ#4NVMr zxdoI@*+yt%p}Tgij8%K=a#6?^OybolTI2ydwYapMc10+8E03?nus$yfZwC}S+MOxyz$+{hQ24}FojDUw9tTjci14PcCg_u}|< zl^gqu#La9V;5xc92MYhVSz&)dDkl?a+$P>UY}d<9=YaL8S|iow*2 zbd!7&ZP+hDy~J1HXxOXly~R%0op!X36jl#hC7?t`J{|ko!Xj}<%S=!>mUxg~cG)4Si${MQ7z!@kDqN3Z>I|C^A#x=()BDxwaVH^lL`gTo^Q!M!<`;Vz{Jkk<%P&^k$dGVu<&|I5 zxHO|JJUJ4*@U4u9=a{3=jEMb|#2sHaN}@lZw?1X4a^Dtr6=78_E0(YZ>isX$HX@G) zSqHV9oeM@`H$xDF`}|#q4})uE6Kpe`HpD2qc z?;HvwVEZq`)%Wilz@2}2Ugq)hYw%hKd;ToeOWw4<^1{z9@mPSRiYE5G3G!Zkr%xO` zHENIVXgBOlecqgkc7%98|HSDfMuMpm`6qAX)%m|Kk~e>Migo>nudgrr6~2K2MG#>? zqW_+?YK4=dyD!L?j3)SJ??{cp!yNv91-XVq6f(-fHZu8IEc!b1D*|5;_=><+1im8h z6@jk^d_~|Z0$&mM{~-b++>)TGN6D`~MLad=m{U-=x^CY~BUx4U@6;T1>V;;%3}JRl z*Izw<`CF=r?(|TjGDJS!yl47{Y3|`4`lMQRZuw;Y{@^0@Pr1V^_aDw*_=~dY4?)+3 z-7L5N?fPl?g_;cK?c465#?%?Ts(~{7o-%C3;UKt< zTzIg)Q6~RL`_6&}bKp5)vZ26Ojn{M`eNK_8L5F~_<*grW~&q-dFb=Bt_6%Moo<<9-nF3Xw^7`gV%BI706mHt^;X z{apa@db|&c-og7!dSY&)&}xL%MR`8^bv4%K&^LczH8CuXINg^3L1^hJ2tCOldK7HO zzeBW2=+YiBc=^tew}+r`;9tE%yn!8llkH0Qx|c;S&0fl}1cqPVS-ES-c z{QtEI{^sr0;ej=1~bH3XKVk4kZ1D=0f+@4q#p|NHZCy4 zU4bU0*kpBRr65=%)35*LiVkW`PaOWtK8rdVOM!jIf%4&TJ$bB;CdG#5X-}iJ?L+>S zS{$4MshAJFo*8ihw10r~?Oh*M;7=pAX~4UzHWhRg{Cwjf57(;rE)$*%6|Y)ke?gt! z0bmKXybl3zJk&qIXDow%0|R+`#?ue(L_nK8X*1MPyjcjDW3mjqJ(RI;>=wRBK{0rF z=)Qa7pQ1?j5zIRDEHE(8_4%R&ZYj)!a_QByZ0wo56%sy}ZPN7DJHLE)NCk~JVQPwb z5N5CyXP%=Irx@`3+}!V|lMSePWt=i^|4W^{k>K?CyN|a}_!9V(-w(%lUwz@o-&8vKct!+A;*3S|B?ta?VPc&Loa{l z{bV$?gy$<8{molWC9o0eG&af66+L%NT116HL z=Np6DGAl96yTWb33iRF#OoU#C_#^W%kH>`LN!P|SO6uIz^;y`n-zW*rdcECU&fPw#d`#`;sZwAjk zR7!8Un9cpQ9Oc44u@h`W%6qA||>eO2@|gZ`$6%Y5yfQ z<~k%(3m^$+Bu++=w{O6mO5{|^ z`$V(C{z`-%07+o)LX}>dscB#*Q}Fq4ym*NB+nfYaqzHUoUd?>No$5M1%^^6|08N46 zyCszy`K(Z?wI%Yk3G7&iz7YBNH}!?5^k6ij7n4Z+C7zkpzpxUsE(@&7yVKM}kr1+_h$`#GlL@3Zx0NavNI~QhAgdQqN-IE(RL}wGxT8-|jPf*o?)nkL-!s=W& z&r|>BZRfssykXa$t)ip(92}kuHy61;8of9G=zIzO4->H}vE!-Etu5kD5o|wZNxeM+ z+-3g?$x7s`wNZ2i>Zk(H4;uF~dih@cUOjKszAIpscZVuI3}SHijZO8$V@;ILB@ubW zepWNA@0KC#`!H^}3e9Dzo5I;a!a+@xwjE<|7rflM)aT(c%`HGv%kGnX_W2zdsJ;dT zv)W4yHco-nkC=hbi*)zW)b}1RVE5?CjGB+&T=m__DzxAE7PGSxx_E?_5G~b&zAHG4 z`oj3ok3Z+jrYp0ZVUalY&SEJ&dHtoWU(4HUPlc;B#)aWjg|kE;ee@!!5-GC)y59F? z@UD#(2D!zgpwx4iNE@1LEv3wpR>P;O4yBD_O>Q{l*?>Cji*XTx1sU$+2e|1u5~}G!5~R z`wcm9ac*5f#<18o4F1#FqsV(4UajmT3{~Pf9O4<;38B1#`5)IIzZV1Wv44O3D6r_> zJ%j%cq^#QCfBT_LNmqeD)4PDYX>oCVQh3tExB$HaT~~!eP{`y zE(M-_alEuo^(r%ljK0R1M8772!)1f&UT*vl2_hFkq#|I}EP#1}bk@C_rZH#j@o>~8 zByr&;mn&X^48a^nzbj0y4a!|LnDc2cYy0WCW7KpZWTVSrq^)%|0ju4L)Vs6nmCGWU zzx~B%2L^HAY&3eB#B6s@{8I<%!nW)f``(=6mOi9MA4S*V%r6UeM4!H5)rj<3w=6@< z&VxZeL1O8%^%wfdDTd;IZNU^}HVdjnGT9^5d?gHxkDlvo$lJXu<4-fAuG!UycO&A%T< z!40ivx_nxGufW*W>QkgzM22F6P~q7W-oW&FTV!||W`bcq z#u0wXQU8|hVn1>g&eSrSNs5yHZh(2T_}crUwg}xGCavfv^1IRpRTJwY=5h%p_!2l` z&H+xqCGQO(4~E?tad83yl#t+vuXmJ0)U)yYBGXjhtQ^1t8$0Mg-cFV$MebrX=_5p_ zKM~)rqFzn}vp>!e1QcbkMyg|yz$)q8U(lLzTAE-;hKzi6Q+_j|4FU(>(<5-26=%H0f{ z9BZrI^pNsH_M%fB%Rnu?FWEuLch(EY$>eiis^UOci3!JPqL`)f;S}YJ1eqjUHpK$D zJjBO;;6g|4M(Kk{MMM+qAYgYvkgm!cTju6t_VL^pP9?%K=zDrDUYj&@jRfo zUvctbN$|-!Pj{xPE;0_UZHQg9dB8th+o`DH(sB?evuOybg0&x z+>e&~F!_`s=Du&>%mUDQ(N{4m$$G>eYay23Tgz5ObF_&6(0(sJXbdk{C9TQ~wA;jy5m59N>iq&Q$b(9W* zU1b9)eK`s5l1eT$PKcM_cCSnwH8Xr&7iFy@cHu@fzJqhM&L67|33mBN7YNHEYS&-GCdHq7{%`QjSV>( z8)_FE6AE7liAW-u6L*;tt-VNAc9xcQmOHi)iFQPyo~_UO|7s9*DkLH-;eR)X44HDn P>?H4}?BoB9{^P#^SB=Vc literal 0 HcmV?d00001 diff --git a/packages/nc-gui/extensions/data-exporter/index.vue b/packages/nc-gui/extensions/data-exporter/index.vue new file mode 100644 index 0000000000..8c42f37ede --- /dev/null +++ b/packages/nc-gui/extensions/data-exporter/index.vue @@ -0,0 +1,180 @@ + + + + + diff --git a/packages/nc-gui/extensions/data-exporter/manifest.json b/packages/nc-gui/extensions/data-exporter/manifest.json new file mode 100644 index 0000000000..60a38170a9 --- /dev/null +++ b/packages/nc-gui/extensions/data-exporter/manifest.json @@ -0,0 +1,11 @@ +{ + "id": "nc-data-exporter", + "title": "Data Exporter", + "description": "Export any view in various formats", + "entry": "data-exporter", + "version": "0.1", + "iconUrl": "data-exporter/icon.png", + "publisherName": "NocoDB", + "publisherEmail": "contact@nocodb.com", + "publisherUrl": "https://www.nocodb.com" +} diff --git a/packages/nocodb/src/controllers/attachments-secure.controller.ts b/packages/nocodb/src/controllers/attachments-secure.controller.ts index d39d310145..9d5a69a8e1 100644 --- a/packages/nocodb/src/controllers/attachments-secure.controller.ts +++ b/packages/nocodb/src/controllers/attachments-secure.controller.ts @@ -69,16 +69,33 @@ export class AttachmentsSecureController { @Get('/dltemp/:param(*)') async fileReadv3(@Param('param') param: string, @Res() res: Response) { try { - const fpath = await PresignedUrl.getPath(`dltemp/${param}`); + const fullPath = await PresignedUrl.getPath(`dltemp/${param}`); + + const queryHelper = fullPath.split('?'); + + const fpath = queryHelper[0]; + + let queryFilename = null; + + if (queryHelper.length > 1) { + const query = new URLSearchParams(queryHelper[1]); + queryFilename = query.get('filename'); + } const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', fpath), }); if (this.attachmentsService.previewAvailable(file.type)) { + if (queryFilename) { + res.setHeader( + 'Content-Disposition', + `attachment; filename=${queryFilename}`, + ); + } res.sendFile(file.path); } else { - res.download(file.path); + res.download(file.path, queryFilename); } } catch (e) { res.status(404).send('Not found'); diff --git a/packages/nocodb/src/controllers/attachments.controller.ts b/packages/nocodb/src/controllers/attachments.controller.ts index 5e6f621545..1ae7d13f8a 100644 --- a/packages/nocodb/src/controllers/attachments.controller.ts +++ b/packages/nocodb/src/controllers/attachments.controller.ts @@ -63,16 +63,26 @@ export class AttachmentsController { // , getCacheMiddleware(), catchError(fileRead)); @Get('/download/:filename(*)') // This route will match any URL that starts with - async fileRead(@Param('filename') filename: string, @Res() res: Response) { + async fileRead( + @Param('filename') filename: string, + @Res() res: Response, + @Query('filename') queryFilename?: string, + ) { try { const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', filename), }); if (this.attachmentsService.previewAvailable(file.type)) { + if (queryFilename) { + res.setHeader( + 'Content-Disposition', + `attachment; filename=${queryFilename}`, + ); + } res.sendFile(file.path); } else { - res.download(file.path); + res.download(file.path, queryFilename); } } catch (e) { res.status(404).send('Not found'); @@ -87,6 +97,7 @@ export class AttachmentsController { @Param('param2') param2: string, @Param('filename') filename: string, @Res() res: Response, + @Query('filename') queryFilename?: string, ) { try { const file = await this.attachmentsService.getFile({ @@ -100,9 +111,15 @@ export class AttachmentsController { }); if (this.attachmentsService.previewAvailable(file.type)) { + if (queryFilename) { + res.setHeader( + 'Content-Disposition', + `attachment; filename=${queryFilename}`, + ); + } res.sendFile(file.path); } else { - res.download(file.path); + res.download(file.path, queryFilename); } } catch (e) { res.status(404).send('Not found'); @@ -112,16 +129,33 @@ export class AttachmentsController { @Get('/dltemp/:param(*)') async fileReadv3(@Param('param') param: string, @Res() res: Response) { try { - const fpath = await PresignedUrl.getPath(`dltemp/${param}`); + const fullPath = await PresignedUrl.getPath(`dltemp/${param}`); + + const queryHelper = fullPath.split('?'); + + const fpath = queryHelper[0]; + + let queryFilename = null; + + if (queryHelper.length > 1) { + const query = new URLSearchParams(queryHelper[1]); + queryFilename = query.get('filename'); + } const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', fpath), }); if (this.attachmentsService.previewAvailable(file.type)) { + if (queryFilename) { + res.setHeader( + 'Content-Disposition', + `attachment; filename=${queryFilename}`, + ); + } res.sendFile(file.path); } else { - res.download(file.path); + res.download(file.path, queryFilename); } } catch (e) { res.status(404).send('Not found'); diff --git a/packages/nocodb/src/controllers/jobs-meta.controller.spec.ts b/packages/nocodb/src/controllers/jobs-meta.controller.spec.ts new file mode 100644 index 0000000000..fa030595a9 --- /dev/null +++ b/packages/nocodb/src/controllers/jobs-meta.controller.spec.ts @@ -0,0 +1,21 @@ +import { Test } from '@nestjs/testing'; +import { HooksService } from '../services/hooks.service'; +import { JobsMetaController } from './jobs-meta.controller'; +import type { TestingModule } from '@nestjs/testing'; + +describe('JobsMetaController', () => { + let controller: JobsMetaController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [JobsMetaController], + providers: [HooksService], + }).compile(); + + controller = module.get(JobsMetaController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/packages/nocodb/src/controllers/jobs-meta.controller.ts b/packages/nocodb/src/controllers/jobs-meta.controller.ts new file mode 100644 index 0000000000..aea3b38618 --- /dev/null +++ b/packages/nocodb/src/controllers/jobs-meta.controller.ts @@ -0,0 +1,28 @@ +import { Body, Controller, Post, Req, UseGuards } from '@nestjs/common'; +import type { JobStatus, JobTypes } from '~/interface/Jobs'; +import { GlobalGuard } from '~/guards/global/global.guard'; +import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; +import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard'; +import { TenantContext } from '~/decorators/tenant-context.decorator'; +import { NcContext, NcRequest } from '~/interface/config'; +import { JobsMetaService } from '~/services/jobs-meta.service'; + +@Controller() +@UseGuards(MetaApiLimiterGuard, GlobalGuard) +export class JobsMetaController { + constructor(private readonly jobsMetaService: JobsMetaService) {} + + @Post(['/api/v2/jobs/:baseId']) + @Acl('jobList') + async jobList( + @TenantContext() context: NcContext, + @Req() req: NcRequest, + @Body() + conditions?: { + job?: JobTypes; + status?: JobStatus; + }, + ) { + return await this.jobsMetaService.list(context, conditions, req); + } +} diff --git a/packages/nocodb/src/helpers/dataHelpers.ts b/packages/nocodb/src/helpers/dataHelpers.ts index 6eb82bd3a3..d0a2a1709f 100644 --- a/packages/nocodb/src/helpers/dataHelpers.ts +++ b/packages/nocodb/src/helpers/dataHelpers.ts @@ -222,6 +222,13 @@ export async function serializeCellValue( .join(', '); } break; + case UITypes.Decimal: + { + if (isNaN(Number(value))) return null; + + return Number(value).toFixed(column.meta?.precision ?? 1); + } + break; default: if (value && typeof value === 'object') { return JSON.stringify(value); diff --git a/packages/nocodb/src/interface/Jobs.ts b/packages/nocodb/src/interface/Jobs.ts index 893bcc56d3..406c55c9dc 100644 --- a/packages/nocodb/src/interface/Jobs.ts +++ b/packages/nocodb/src/interface/Jobs.ts @@ -1,4 +1,5 @@ -import type { NcContext } from '~/interface/config'; +import type { UserType } from 'nocodb-sdk'; +import type { NcContext, NcRequest } from '~/interface/config'; export const JOBS_QUEUE = 'jobs'; export enum JobTypes { @@ -15,6 +16,7 @@ export enum JobTypes { HealthCheck = 'health-check', HandleWebhook = 'handle-webhook', CleanUp = 'clean-up', + DataExport = 'data-export', } export enum JobStatus { @@ -44,12 +46,77 @@ export enum InstanceCommands { RELEASE = 'release', } -export interface HandleWebhookJobData { +export interface JobData { context: NcContext; + user: Partial; +} + +export interface AtImportJobData extends JobData { + syncId: string; + baseId: string; + sourceId: string; + baseName: string; + authToken: string; + baseURL: string; + clientIp: string; + options?: { + syncViews?: boolean; + syncAttachment?: boolean; + syncLookup?: boolean; + syncRollup?: boolean; + syncUsers?: boolean; + syncData?: boolean; + }; + user: any; +} + +export interface DuplicateBaseJobData extends JobData { + sourceId: string; + dupProjectId: string; + req: NcRequest; + options: { + excludeData?: boolean; + excludeViews?: boolean; + excludeHooks?: boolean; + }; +} + +export interface DuplicateModelJobData extends JobData { + sourceId: string; + modelId: string; + title: string; + req: NcRequest; + options: { + excludeData?: boolean; + excludeViews?: boolean; + excludeHooks?: boolean; + }; +} + +export interface DuplicateColumnJobData extends JobData { + sourceId: string; + columnId: string; + extra: Record; // extra data + req: NcRequest; + options: { + excludeData?: boolean; + }; +} + +export interface HandleWebhookJobData extends JobData { hookId: string; modelId: string; viewId: string; prevData; newData; - user; +} + +export interface DataExportJobData extends JobData { + options?: { + delimiter?: string; + }; + modelId: string; + viewId: string; + exportAs: 'csv' | 'json' | 'xlsx'; + ncSiteUrl: string; } diff --git a/packages/nocodb/src/meta/meta.service.ts b/packages/nocodb/src/meta/meta.service.ts index 7eba9a7215..25582842db 100644 --- a/packages/nocodb/src/meta/meta.service.ts +++ b/packages/nocodb/src/meta/meta.service.ts @@ -251,6 +251,7 @@ export class MetaService { [MetaTable.COMMENTS]: 'com', [MetaTable.COMMENTS_REACTIONS]: 'cre', [MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE]: 'cnp', + [MetaTable.JOBS]: 'job', }; const prefix = prefixMap[target] || 'nc'; @@ -726,6 +727,16 @@ export class MetaService { ); } + public formatDateTime(date: string): string { + return dayjs(date) + .utc() + .format( + this.isMySQL() || this.isMssql() + ? 'YYYY-MM-DD HH:mm:ss' + : 'YYYY-MM-DD HH:mm:ssZ', + ); + } + public async init(): Promise { await this.connection.migrate.latest({ migrationSource: new XcMigrationSource(), diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index d8733403a0..b6f946ced4 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -39,6 +39,7 @@ import * as nc_049_clear_notifications from '~/meta/migrations/v2/nc_049_clear_n import * as nc_050_tenant_isolation from '~/meta/migrations/v2/nc_050_tenant_isolation'; import * as nc_051_source_readonly_columns from '~/meta/migrations/v2/nc_051_source_readonly_columns'; import * as nc_052_field_aggregation from '~/meta/migrations/v2/nc_052_field_aggregation'; +import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -89,6 +90,7 @@ export default class XcMigrationSourcev2 { 'nc_050_tenant_isolation', 'nc_051_source_readonly_columns', 'nc_052_field_aggregation', + 'nc_053_jobs', ]); } @@ -180,6 +182,8 @@ export default class XcMigrationSourcev2 { return nc_051_source_readonly_columns; case 'nc_052_field_aggregation': return nc_052_field_aggregation; + case 'nc_053_jobs': + return nc_053_jobs; } } } diff --git a/packages/nocodb/src/meta/migrations/v2/nc_053_jobs.ts b/packages/nocodb/src/meta/migrations/v2/nc_053_jobs.ts new file mode 100644 index 0000000000..13580eaa26 --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_053_jobs.ts @@ -0,0 +1,30 @@ +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.createTable(MetaTable.JOBS, (table) => { + table.string('id', 20).primary(); + + table.string('job', 255); + + table.string('status', 20); + + table.text('result'); + + table.string('fk_user_id', 20); + + table.string('fk_workspace_id', 20); + + table.string('base_id', 20); + + table.timestamps(true, true); + + // TODO - add indexes + }); +}; + +const down = async (knex: Knex) => { + await knex.schema.dropTable(MetaTable.JOBS); +}; + +export { up, down }; diff --git a/packages/nocodb/src/models/Job.ts b/packages/nocodb/src/models/Job.ts new file mode 100644 index 0000000000..e1b63f12a1 --- /dev/null +++ b/packages/nocodb/src/models/Job.ts @@ -0,0 +1,136 @@ +import type { NcContext } from '~/interface/config'; +import type { Condition } from '~/db/CustomKnex'; +import Noco from '~/Noco'; +import { + CacheDelDirection, + CacheGetType, + CacheScope, + MetaTable, +} from '~/utils/globals'; +import NocoCache from '~/cache/NocoCache'; +import { extractProps } from '~/helpers/extractProps'; +import { prepareForDb, prepareForResponse } from '~/utils/modelUtils'; + +export default class Job { + id: string; + job: string; + status: string; + result: string; + fk_user_id: string; + fk_workspace_id: string; + base_id: string; + created_at: Date; + updated_at: Date; + + constructor(data: Partial) { + Object.assign(this, data); + } + + public static async insert( + context: NcContext, + jobObj: Partial, + ncMeta = Noco.ncMeta, + ) { + const insertObj = extractProps(jobObj, [ + 'job', + 'status', + 'result', + 'fk_user_id', + ]); + + const { id } = await ncMeta.metaInsert2( + context.workspace_id, + context.base_id, + MetaTable.JOBS, + insertObj, + ); + + return this.get(context, id, ncMeta); + } + + public static async update( + context: NcContext, + jobId: string, + jobObj: Partial, + ncMeta = Noco.ncMeta, + ) { + const updateObj = extractProps(jobObj, ['status', 'result']); + + const res = await ncMeta.metaUpdate( + context.workspace_id, + context.base_id, + MetaTable.JOBS, + prepareForDb(updateObj, 'result'), + jobId, + ); + + await NocoCache.update( + `${CacheScope.JOBS}:${jobId}`, + prepareForResponse(updateObj, 'result'), + ); + + return res; + } + + public static async delete( + context: NcContext, + jobId: string, + ncMeta = Noco.ncMeta, + ) { + await ncMeta.metaDelete( + context.workspace_id, + context.base_id, + MetaTable.JOBS, + jobId, + ); + + await NocoCache.deepDel( + `${CacheScope.JOBS}:${jobId}`, + CacheDelDirection.CHILD_TO_PARENT, + ); + } + + public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) { + let jobData = + id && + (await NocoCache.get( + `${CacheScope.JOBS}:${id}`, + CacheGetType.TYPE_OBJECT, + )); + + if (!jobData) { + jobData = await ncMeta.metaGet2( + context.workspace_id, + context.base_id, + MetaTable.JOBS, + id, + ); + + jobData = prepareForResponse(jobData, 'result'); + + await NocoCache.set(`${CacheScope.JOBS}:${id}`, jobData); + } + + return jobData && new Job(jobData); + } + + public static async list( + context: NcContext, + opts: { + condition?: Record; + xcCondition?: Condition; + }, + ncMeta = Noco.ncMeta, + ): Promise { + const jobList = await ncMeta.metaList2( + context.workspace_id, + context.base_id, + MetaTable.JOBS, + opts, + ); + + return jobList.map((job) => { + return new Job(prepareForResponse(job, 'result')); + }); + } +} diff --git a/packages/nocodb/src/models/PresignedUrl.ts b/packages/nocodb/src/models/PresignedUrl.ts index 1e9213f31a..a344997263 100644 --- a/packages/nocodb/src/models/PresignedUrl.ts +++ b/packages/nocodb/src/models/PresignedUrl.ts @@ -91,10 +91,18 @@ export default class PresignedUrl { path: string; expireSeconds?: number; s3?: boolean; + filename?: string; }, ncMeta = Noco.ncMeta, ) { - const { path, expireSeconds = DEFAULT_EXPIRE_SECONDS, s3 = false } = param; + let { path } = param; + + const { + expireSeconds = DEFAULT_EXPIRE_SECONDS, + s3 = false, + filename, + } = param; + const expireAt = roundExpiry( new Date(new Date().getTime() + expireSeconds * 1000), ); // at least expireSeconds from now @@ -129,6 +137,7 @@ export default class PresignedUrl { tempUrl = await (storageAdapter as any).getSignedUrl( path, expiresInSeconds, + filename, ); await this.add({ path: path, @@ -139,6 +148,12 @@ export default class PresignedUrl { } else { // if not present, create a new url tempUrl = `dltemp/${nanoid(16)}/${expireAt.getTime()}/${path}`; + + // if filename is present, add it to the destination + if (filename) { + path = `${path}?filename=${encodeURIComponent(filename)}`; + } + await this.add({ path: path, url: tempUrl, diff --git a/packages/nocodb/src/models/index.ts b/packages/nocodb/src/models/index.ts index ae0c876191..e79c8f07ec 100644 --- a/packages/nocodb/src/models/index.ts +++ b/packages/nocodb/src/models/index.ts @@ -43,3 +43,4 @@ export { default as PresignedUrl } from './PresignedUrl'; export { default as UserRefreshToken } from './UserRefreshToken'; export { default as Extension } from './Extension'; export { default as Comment } from './Comment'; +export { default as Job } from './Job'; diff --git a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts index 04b47e041a..37b8323477 100644 --- a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts @@ -7,7 +7,8 @@ import { MetaSyncProcessor } from '~/modules/jobs/jobs/meta-sync/meta-sync.proce import { SourceCreateProcessor } from '~/modules/jobs/jobs/source-create/source-create.processor'; import { SourceDeleteProcessor } from '~/modules/jobs/jobs/source-delete/source-delete.processor'; import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/webhook-handler.processor'; -import { JobsEventService } from '~/modules/jobs/fallback/jobs-event.service'; +import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor'; +import { JobsEventService } from '~/modules/jobs/jobs-event.service'; import { JobStatus, JobTypes } from '~/interface/Jobs'; export interface Job { @@ -33,6 +34,7 @@ export class QueueService { protected readonly sourceCreateProcessor: SourceCreateProcessor, protected readonly sourceDeleteProcessor: SourceDeleteProcessor, protected readonly webhookHandlerProcessor: WebhookHandlerProcessor, + protected readonly dataExportProcessor: DataExportProcessor, ) { this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => { const job = this.queueMemory.find((job) => job.id === data.job.id); @@ -94,6 +96,10 @@ export class QueueService { this: this.webhookHandlerProcessor, fn: this.webhookHandlerProcessor.job, }, + [JobTypes.DataExport]: { + this: this.dataExportProcessor, + fn: this.dataExportProcessor.job, + }, }; async jobWrapper(job: Job) { @@ -129,8 +135,8 @@ export class QueueService { QueueService.queueIdCounter = index; } - add(name: string, data: any, _opts = {}) { - const id = `${this.queueIndex++}`; + add(name: string, data: any, opts?: { jobId?: string }) { + const id = opts?.jobId || `${this.queueIndex++}`; const job = { id: `${id}`, name, status: JobStatus.WAITING, data }; this.queueMemory.push(job); this.queue.add(() => this.jobWrapper(job)); diff --git a/packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts b/packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts deleted file mode 100644 index 9abbb7ae37..0000000000 --- a/packages/nocodb/src/modules/jobs/fallback/jobs-event.service.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { - OnQueueActive, - OnQueueCompleted, - OnQueueFailed, - Processor, -} from '@nestjs/bull'; -import { Job } from 'bull'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { Logger } from '@nestjs/common'; -import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; - -@Processor(JOBS_QUEUE) -export class JobsEventService { - protected logger = new Logger(JobsEventService.name); - - constructor(private eventEmitter: EventEmitter2) {} - - @OnQueueActive() - onActive(job: Job) { - this.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.ACTIVE, - }); - } - - @OnQueueFailed() - onFailed(job: Job, error: Error) { - this.logger.error( - `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, - ); - - const newLocal = this; - newLocal.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.FAILED, - data: { - error: { - message: error?.message, - }, - }, - }); - } - - @OnQueueCompleted() - onCompleted(job: Job, data: any) { - this.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.COMPLETED, - data: { - result: data, - }, - }); - } -} diff --git a/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts index c8f1676723..0b42c98266 100644 --- a/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts @@ -2,6 +2,8 @@ import { Injectable } from '@nestjs/common'; import type { OnModuleInit } from '@nestjs/common'; import { QueueService } from '~/modules/jobs/fallback/fallback-queue.service'; import { JobStatus } from '~/interface/Jobs'; +import { Job } from '~/models'; +import { RootScopes } from '~/utils/globals'; @Injectable() export class JobsService implements OnModuleInit { @@ -10,7 +12,21 @@ export class JobsService implements OnModuleInit { async onModuleInit() {} async add(name: string, data: any) { - return this.fallbackQueueService.add(name, data); + const context = { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + ...(data?.context || {}), + }; + + const jobData = await Job.insert(context, { + job: name, + status: JobStatus.WAITING, + fk_user_id: data?.user?.id, + }); + + this.fallbackQueueService.add(name, data, { jobId: jobData.id }); + + return jobData; } async jobStatus(jobId: string) { @@ -28,30 +44,6 @@ export class JobsService implements OnModuleInit { ]); } - async getJobWithData(data: any) { - const jobs = await this.fallbackQueueService.getJobs([ - // 'completed', - JobStatus.WAITING, - JobStatus.ACTIVE, - JobStatus.DELAYED, - // 'failed', - JobStatus.PAUSED, - ]); - - const job = jobs.find((j) => { - for (const key in data) { - if (j.data[key]) { - if (j.data[key] !== data[key]) return false; - } else { - return false; - } - } - return true; - }); - - return job; - } - async resumeQueue() { await this.fallbackQueueService.queue.start(); } diff --git a/packages/nocodb/src/modules/jobs/jobs-event.service.ts b/packages/nocodb/src/modules/jobs/jobs-event.service.ts new file mode 100644 index 0000000000..2badfa1122 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs-event.service.ts @@ -0,0 +1,108 @@ +import { + OnQueueActive, + OnQueueCompleted, + OnQueueFailed, + Processor, +} from '@nestjs/bull'; +import { Job as BullJob } from 'bull'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Logger } from '@nestjs/common'; +import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; +import { Job } from '~/models'; +import { RootScopes } from '~/utils/globals'; + +@Processor(JOBS_QUEUE) +export class JobsEventService { + protected logger = new Logger(JobsEventService.name); + + constructor(private eventEmitter: EventEmitter2) {} + + @OnQueueActive() + onActive(job: BullJob) { + Job.update( + { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + }, + job.id.toString(), + { + status: JobStatus.ACTIVE, + }, + ) + .then(() => { + this.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.ACTIVE, + }); + }) + .catch((error) => { + this.logger.error( + `Failed to update job (${job.id}) status to active: ${error.message}`, + ); + }); + } + + @OnQueueFailed() + onFailed(job: BullJob, error: Error) { + this.logger.error( + `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, + ); + + Job.update( + { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + }, + job.id.toString(), + { + status: JobStatus.FAILED, + }, + ) + .then(() => { + const newLocal = this; + newLocal.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.FAILED, + data: { + error: { + message: error?.message, + }, + }, + }); + }) + .catch((error) => { + this.logger.error( + `Failed to update job (${job.id}) status to failed: ${error.message}`, + ); + }); + } + + @OnQueueCompleted() + onCompleted(job: BullJob, data: any) { + Job.update( + { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + }, + job.id.toString(), + { + status: JobStatus.COMPLETED, + result: data, + }, + ) + .then(() => { + this.eventEmitter.emit(JobEvents.STATUS, { + id: job.id.toString(), + status: JobStatus.COMPLETED, + data: { + result: data, + }, + }); + }) + .catch((error) => { + this.logger.error( + `Failed to update job (${job.id}) status to completed: ${error.message}`, + ); + }); + } +} diff --git a/packages/nocodb/src/modules/jobs/jobs-service.interface.ts b/packages/nocodb/src/modules/jobs/jobs-service.interface.ts index 7f083582f2..3696067b71 100644 --- a/packages/nocodb/src/modules/jobs/jobs-service.interface.ts +++ b/packages/nocodb/src/modules/jobs/jobs-service.interface.ts @@ -7,7 +7,6 @@ export interface IJobsService { add(name: string, data: any): Promise>; jobStatus(jobId: string): Promise; jobList(): Promise[]>; - getJobWithData(data: any): Promise>; resumeQueue(): Promise; pauseQueue(): Promise; } diff --git a/packages/nocodb/src/modules/jobs/jobs.controller.ts b/packages/nocodb/src/modules/jobs/jobs.controller.ts index c363544730..045cc60da2 100644 --- a/packages/nocodb/src/modules/jobs/jobs.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs.controller.ts @@ -56,7 +56,7 @@ export class JobsController { } else { messages = ( await NocoCache.get( - `${CacheScope.JOBS}:${jobId}:messages`, + `${CacheScope.JOBS_POLLING}:${jobId}:messages`, CacheGetType.TYPE_OBJECT, ) )?.messages; @@ -92,38 +92,43 @@ export class JobsController { }; // subscribe to job events if (JobsRedis.available) { - const unsubscribeCallback = await JobsRedis.subscribe(jobId, async (data) => { - if (this.jobRooms[jobId]) { - this.jobRooms[jobId].listeners.forEach((res) => { - if (!res.headersSent) { - res.send({ - status: 'refresh', - }); - } - }); - } - - const cmd = data.cmd; - delete data.cmd; - switch (cmd) { - case JobEvents.STATUS: - if ( - [JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status) - ) { - await unsubscribeCallback(); - delete this.jobRooms[jobId]; - // close the job after 1 second (to allow the update of messages) - setTimeout(() => { - this.closedJobs.push(jobId); - }, 1000); - // remove the job after polling interval * 2 - setTimeout(() => { - this.closedJobs = this.closedJobs.filter((j) => j !== jobId); - }, POLLING_INTERVAL * 2); - } - break; - } - }); + const unsubscribeCallback = await JobsRedis.subscribe( + jobId, + async (data) => { + if (this.jobRooms[jobId]) { + this.jobRooms[jobId].listeners.forEach((res) => { + if (!res.headersSent) { + res.send({ + status: 'refresh', + }); + } + }); + } + + const cmd = data.cmd; + delete data.cmd; + switch (cmd) { + case JobEvents.STATUS: + if ( + [JobStatus.COMPLETED, JobStatus.FAILED].includes(data.status) + ) { + await unsubscribeCallback(); + delete this.jobRooms[jobId]; + // close the job after 1 second (to allow the update of messages) + setTimeout(() => { + this.closedJobs.push(jobId); + }, 1000); + // remove the job after polling interval * 2 + setTimeout(() => { + this.closedJobs = this.closedJobs.filter( + (j) => j !== jobId, + ); + }, POLLING_INTERVAL * 2); + } + break; + } + }, + ); } } @@ -144,32 +149,6 @@ export class JobsController { }, POLLING_INTERVAL); } - @Post('/jobs/status') - async status(@Body() data: { id: string } | any) { - let res: { - id?: string; - status?: JobStatus; - } | null = null; - if (Object.keys(data).every((k) => ['id'].includes(k)) && data?.id) { - const rooms = (await this.jobsService.jobList()).map( - (j) => `jobs-${j.id}`, - ); - const room = rooms.find((r) => r === `jobs-${data.id}`); - if (room) { - res.id = data.id; - } - } else { - const job = await this.jobsService.getJobWithData(data); - if (job) { - res = {}; - res.id = `${job.id}`; - res.status = await this.jobsService.jobStatus(data.id); - } - } - - return res; - } - @OnEvent(JobEvents.STATUS) async sendJobStatus(data: { id: string; @@ -193,7 +172,7 @@ export class JobsController { this.localJobs[jobId].messages.shift(); } - await NocoCache.set(`${CacheScope.JOBS}:${jobId}:messages`, { + await NocoCache.set(`${CacheScope.JOBS_POLLING}:${jobId}:messages`, { messages: this.localJobs[jobId].messages, }); } else { @@ -208,7 +187,7 @@ export class JobsController { _mid: 1, }; - await NocoCache.set(`${CacheScope.JOBS}:${jobId}:messages`, { + await NocoCache.set(`${CacheScope.JOBS_POLLING}:${jobId}:messages`, { messages: this.localJobs[jobId].messages, }); } @@ -237,7 +216,7 @@ export class JobsController { setTimeout(async () => { delete this.jobRooms[jobId]; delete this.localJobs[jobId]; - await NocoCache.del(`${CacheScope.JOBS}:${jobId}:messages`); + await NocoCache.del(`${CacheScope.JOBS_POLLING}:${jobId}:messages`); }, POLLING_INTERVAL * 2); } } @@ -265,7 +244,7 @@ export class JobsController { this.localJobs[jobId].messages.shift(); } - await NocoCache.set(`${CacheScope.JOBS}:${jobId}:messages`, { + await NocoCache.set(`${CacheScope.JOBS_POLLING}:${jobId}:messages`, { messages: this.localJobs[jobId].messages, }); } else { @@ -280,7 +259,7 @@ export class JobsController { _mid: 1, }; - await NocoCache.set(`${CacheScope.JOBS}:${jobId}:messages`, { + await NocoCache.set(`${CacheScope.JOBS_POLLING}:${jobId}:messages`, { messages: this.localJobs[jobId].messages, }); } diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index 6f26ad3ddf..6f718d7efe 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -16,18 +16,19 @@ import { SourceCreateProcessor } from '~/modules/jobs/jobs/source-create/source- import { SourceDeleteController } from '~/modules/jobs/jobs/source-delete/source-delete.controller'; import { SourceDeleteProcessor } from '~/modules/jobs/jobs/source-delete/source-delete.processor'; import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/webhook-handler.processor'; +import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor'; +import { DataExportController } from '~/modules/jobs/jobs/data-export/data-export.controller'; // Jobs Module Related import { JobsLogService } from '~/modules/jobs/jobs/jobs-log.service'; // import { JobsGateway } from '~/modules/jobs/jobs.gateway'; import { JobsController } from '~/modules/jobs/jobs.controller'; import { JobsService } from '~/modules/jobs/redis/jobs.service'; -import { JobsEventService } from '~/modules/jobs/redis/jobs-event.service'; +import { JobsEventService } from '~/modules/jobs/jobs-event.service'; // Fallback import { JobsService as FallbackJobsService } from '~/modules/jobs/fallback/jobs.service'; import { QueueService as FallbackQueueService } from '~/modules/jobs/fallback/fallback-queue.service'; -import { JobsEventService as FallbackJobsEventService } from '~/modules/jobs/fallback/jobs-event.service'; import { JOBS_QUEUE } from '~/interface/Jobs'; export const JobsModuleMetadata = { @@ -53,14 +54,14 @@ export const JobsModuleMetadata = { MetaSyncController, SourceCreateController, SourceDeleteController, + DataExportController, ] : []), ], providers: [ ...(process.env.NC_WORKER_CONTAINER !== 'true' ? [] : []), - ...(process.env.NC_REDIS_JOB_URL - ? [JobsEventService] - : [FallbackQueueService, FallbackJobsEventService]), + JobsEventService, + ...(process.env.NC_REDIS_JOB_URL ? [] : [FallbackQueueService]), { provide: 'JobsService', useClass: process.env.NC_REDIS_JOB_URL @@ -76,6 +77,7 @@ export const JobsModuleMetadata = { SourceCreateProcessor, SourceDeleteProcessor, WebhookHandlerProcessor, + DataExportProcessor, ], exports: ['JobsService'], }; diff --git a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts index e21f6ca9c0..e011cff638 100644 --- a/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts @@ -31,7 +31,7 @@ import { TablesService } from '~/services/tables.service'; import { ViewColumnsService } from '~/services/view-columns.service'; import { ViewsService } from '~/services/views.service'; import { FormsService } from '~/services/forms.service'; -import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs'; +import { AtImportJobData, JOBS_QUEUE, JobTypes } from '~/interface/Jobs'; import { GridColumnsService } from '~/services/grid-columns.service'; import { TelemetryService } from '~/services/telemetry.service'; import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; @@ -112,7 +112,7 @@ export class AtImportProcessor { ) {} @Process(JobTypes.AtImport) - async job(job: Job) { + async job(job: Job) { this.debugLog(`job started for ${job.id}`); const context = job.data.context; @@ -2668,7 +2668,7 @@ export interface AirtableSyncConfig { apiKey: string; appId?: string; shareId: string; - user: UserType; + user: Partial; options: { syncViews: boolean; syncData: boolean; diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.controller.ts b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.controller.ts new file mode 100644 index 0000000000..f754a1677b --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.controller.ts @@ -0,0 +1,58 @@ +import { + Body, + Controller, + HttpCode, + Inject, + Param, + Post, + Req, + UseGuards, +} from '@nestjs/common'; +import type { DataExportJobData } from '~/interface/Jobs'; +import { GlobalGuard } from '~/guards/global/global.guard'; +import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; +import { BasesService } from '~/services/bases.service'; +import { View } from '~/models'; +import { JobTypes } from '~/interface/Jobs'; +import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard'; +import { IJobsService } from '~/modules/jobs/jobs-service.interface'; +import { TenantContext } from '~/decorators/tenant-context.decorator'; +import { NcContext, NcRequest } from '~/interface/config'; +import { NcError } from '~/helpers/catchError'; + +@Controller() +@UseGuards(MetaApiLimiterGuard, GlobalGuard) +export class DataExportController { + constructor( + @Inject('JobsService') protected readonly jobsService: IJobsService, + protected readonly basesService: BasesService, + ) {} + + @Post(['/api/v2/export/:viewId/:exportAs']) + @HttpCode(200) + // TODO add new ACL + @Acl('dataList') + async exportModelData( + @TenantContext() context: NcContext, + @Req() req: NcRequest, + @Param('viewId') viewId: string, + @Param('exportAs') exportAs: 'csv' | 'json' | 'xlsx', + @Body() options: DataExportJobData['options'], + ) { + const view = await View.get(context, viewId); + + if (!view) NcError.viewNotFound(viewId); + + const job = await this.jobsService.add(JobTypes.DataExport, { + context, + options, + modelId: view.fk_model_id, + viewId, + user: req.user, + exportAs, + ncSiteUrl: req.ncSiteUrl, + }); + + return job; + } +} diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts new file mode 100644 index 0000000000..4a76bf0c78 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts @@ -0,0 +1,132 @@ +import { Readable } from 'stream'; +import path from 'path'; +import { Process, Processor } from '@nestjs/bull'; +import { Logger } from '@nestjs/common'; +import { Job } from 'bull'; +import moment from 'moment'; +import { type DataExportJobData, JOBS_QUEUE, JobTypes } from '~/interface/Jobs'; +import { elapsedTime, initTime } from '~/modules/jobs/helpers'; +import { ExportService } from '~/modules/jobs/jobs/export-import/export.service'; +import { Model, PresignedUrl, View } from '~/models'; +import { NcError } from '~/helpers/catchError'; +import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; + +function getViewTitle(view: View) { + return view.is_default ? 'Default View' : view.title; +} + +@Processor(JOBS_QUEUE) +export class DataExportProcessor { + private logger = new Logger(DataExportProcessor.name); + + constructor(private readonly exportService: ExportService) {} + + @Process(JobTypes.DataExport) + async job(job: Job) { + const { + context, + options, + modelId, + viewId, + user: _user, + exportAs, + ncSiteUrl, + } = job.data; + + if (exportAs !== 'csv') NcError.notImplemented(`Export as ${exportAs}`); + + const hrTime = initTime(); + + const model = await Model.get(context, modelId); + + if (!model) NcError.tableNotFound(modelId); + + const view = await View.get(context, viewId); + + if (!view) NcError.viewNotFound(viewId); + + // date time as containing folder YYYY-MM-DD/HH + const dateFolder = moment().format('YYYY-MM-DD/HH'); + + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + const destPath = `nc/uploads/data-export/${dateFolder}/${modelId}/${ + model.title + } (${getViewTitle(view)}) - ${Date.now()}.csv`; + + let url = null; + + try { + const dataStream = new Readable({ + read() {}, + }); + + dataStream.setEncoding('utf8'); + + let error = null; + + const uploadFilePromise = (storageAdapter as any) + .fileCreateByStream(destPath, dataStream) + .catch((e) => { + this.logger.error(e); + error = e; + }); + + this.exportService + .streamModelDataAsCsv(context, { + dataStream, + linkStream: null, + baseId: model.base_id, + modelId: model.id, + viewId: view.id, + ncSiteUrl: ncSiteUrl, + delimiter: options?.delimiter, + }) + .catch((e) => { + this.logger.debug(e); + dataStream.push(null); + error = e; + }); + + url = await uploadFilePromise; + + // if url is not defined, it is local attachment + if (!url) { + url = await PresignedUrl.getSignedUrl({ + path: path.join(destPath.replace('nc/uploads/', '')), + filename: `${model.title} (${getViewTitle(view)}).csv`, + expireSeconds: 3 * 60 * 60, // 3 hours + }); + } else { + if (url.includes('.amazonaws.com/')) { + const relativePath = decodeURI(url.split('.amazonaws.com/')[1]); + url = await PresignedUrl.getSignedUrl({ + path: relativePath, + filename: `${model.title} (${getViewTitle(view)}).csv`, + s3: true, + expireSeconds: 3 * 60 * 60, // 3 hours + }); + } + } + + if (error) { + throw error; + } + + elapsedTime( + hrTime, + `exported data for model ${modelId} view ${viewId} as ${exportAs}`, + 'exportData', + ); + } catch (e) { + throw NcError.badRequest(e); + } + + return { + timestamp: new Date(), + type: exportAs, + title: `${model.title} (${getViewTitle(view)})`, + url, + }; + } +} diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts index 9984b757c5..05f4cdc987 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.controller.ts @@ -8,10 +8,7 @@ import { Req, UseGuards, } from '@nestjs/common'; -import { - ProjectStatus, - readonlyMetaAllowedTypes, -} from 'nocodb-sdk'; +import { ProjectStatus, readonlyMetaAllowedTypes } from 'nocodb-sdk'; import { GlobalGuard } from '~/guards/global/global.guard'; import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; import { BasesService } from '~/services/bases.service'; @@ -95,6 +92,7 @@ export class DuplicateController { workspace_id: base.fk_workspace_id, base_id: base.id, }, + user: req.user, baseId: base.id, sourceId: source.id, dupProjectId: dupProject.id, @@ -168,6 +166,7 @@ export class DuplicateController { const job = await this.jobsService.add(JobTypes.DuplicateBase, { context, + user: req.user, baseId: base.id, sourceId: source.id, dupProjectId: dupProject.id, @@ -233,6 +232,7 @@ export class DuplicateController { const job = await this.jobsService.add(JobTypes.DuplicateModel, { context, + user: req.user, baseId: base.id, sourceId: source.id, modelId: model.id, @@ -302,6 +302,7 @@ export class DuplicateController { const job = await this.jobsService.add(JobTypes.DuplicateColumn, { context, + user: req.user, baseId: base.id, sourceId: column.source_id, modelId: model.id, diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts index ff60b6dc5c..be8c647033 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts @@ -5,6 +5,11 @@ import papaparse from 'papaparse'; import debug from 'debug'; import { isLinksOrLTAR, isVirtualCol, RelationTypes } from 'nocodb-sdk'; import type { NcContext } from '~/interface/config'; +import type { + DuplicateBaseJobData, + DuplicateColumnJobData, + DuplicateModelJobData, +} from '~/interface/Jobs'; import { Base, Column, Model, Source } from '~/models'; import { BasesService } from '~/services/bases.service'; import { @@ -31,7 +36,7 @@ export class DuplicateProcessor { ) {} @Process(JobTypes.DuplicateBase) - async duplicateBase(job: Job) { + async duplicateBase(job: Job) { this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateBase})`); const hrTime = initTime(); @@ -131,10 +136,12 @@ export class DuplicateProcessor { } this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateBase})`); + + return { id: dupProject.id }; } @Process(JobTypes.DuplicateModel) - async duplicateModel(job: Job) { + async duplicateModel(job: Job) { this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateModel})`); const hrTime = initTime(); @@ -241,11 +248,11 @@ export class DuplicateProcessor { this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`); - return await Model.get(context, findWithIdentifier(idMap, sourceModel.id)); + return { id: findWithIdentifier(idMap, sourceModel.id) }; } @Process(JobTypes.DuplicateColumn) - async duplicateColumn(job: Job) { + async duplicateColumn(job: Job) { this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateColumn})`); const hrTime = initTime(); @@ -398,10 +405,7 @@ export class DuplicateProcessor { this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`); - return await Column.get(context, { - source_id: base.id, - colId: findWithIdentifier(idMap, sourceColumn.id), - }); + return { id: findWithIdentifier(idMap, sourceColumn.id) }; } async importModelsData( diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts index 2866fb01de..05e063961c 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts @@ -9,7 +9,10 @@ import type { NcContext } from '~/interface/config'; import type { LinkToAnotherRecordColumn } from '~/models'; import { Base, Filter, Hook, Model, Source, View } from '~/models'; import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; -import { getViewAndModelByAliasOrId } from '~/helpers/dataHelpers'; +import { + getViewAndModelByAliasOrId, + serializeCellValue, +} from '~/helpers/dataHelpers'; import { clearPrefix, generateBaseIdMap, @@ -447,10 +450,14 @@ export class ExportService { viewId?: string; handledMmList?: string[]; _fieldIds?: string[]; + ncSiteUrl?: string; + delimiter?: string; }, ) { const { dataStream, linkStream, handledMmList } = param; + const dataExportMode = !linkStream; + const { model, view } = await getViewAndModelByAliasOrId(context, { baseName: param.baseId, tableName: param.modelId, @@ -463,32 +470,35 @@ export class ExportService { const btMap = new Map(); - for (const column of model.columns.filter( - (col) => - col.uidt === UITypes.LinkToAnotherRecord && - (col.colOptions?.type === RelationTypes.BELONGS_TO || - (col.colOptions?.type === RelationTypes.ONE_TO_ONE && col.meta?.bt)), - )) { - await column.getColOptions(context); - const fkCol = model.columns.find( - (c) => c.id === column.colOptions?.fk_child_column_id, - ); - if (fkCol) { - // replace bt column with fk column if it is in _fieldIds - if (param._fieldIds && param._fieldIds.includes(column.id)) { - param._fieldIds.push(fkCol.id); - const btIndex = param._fieldIds.indexOf(column.id); - param._fieldIds.splice(btIndex, 1); - } - - btMap.set( - fkCol.id, - `${column.base_id}::${column.source_id}::${column.fk_model_id}::${column.id}`, + if (!dataExportMode) { + for (const column of model.columns.filter( + (col) => + col.uidt === UITypes.LinkToAnotherRecord && + (col.colOptions?.type === RelationTypes.BELONGS_TO || + (col.colOptions?.type === RelationTypes.ONE_TO_ONE && + col.meta?.bt)), + )) { + await column.getColOptions(context); + const fkCol = model.columns.find( + (c) => c.id === column.colOptions?.fk_child_column_id, ); + if (fkCol) { + // replace bt column with fk column if it is in _fieldIds + if (param._fieldIds && param._fieldIds.includes(column.id)) { + param._fieldIds.push(fkCol.id); + const btIndex = param._fieldIds.indexOf(column.id); + param._fieldIds.splice(btIndex, 1); + } + + btMap.set( + fkCol.id, + `${column.base_id}::${column.source_id}::${column.fk_model_id}::${column.id}`, + ); + } } } - const fields = param._fieldIds + let fields = param._fieldIds ? model.columns .filter((c) => param._fieldIds?.includes(c.id)) .map((c) => c.title) @@ -498,6 +508,16 @@ export class ExportService { .map((c) => c.title) .join(','); + if (dataExportMode) { + const viewCols = await view.getColumns(context); + + fields = viewCols + .sort((a, b) => a.order - b.order) + .filter((c) => c.show) + .map((vc) => model.columns.find((c) => c.id === vc.fk_column_id).title) + .join(','); + } + const mmColumns = param._fieldIds ? model.columns .filter((c) => param._fieldIds?.includes(c.id)) @@ -506,7 +526,7 @@ export class ExportService { (col) => isLinksOrLTAR(col) && col.colOptions?.type === 'mm', ); - const hasLink = mmColumns.length > 0; + const hasLink = !dataExportMode && mmColumns.length > 0; dataStream.setEncoding('utf8'); @@ -564,6 +584,22 @@ export class ExportService { return { data }; }; + const formatAndSerialize = async (data: any) => { + for (const row of data) { + for (const [k, v] of Object.entries(row)) { + const col = model.columns.find((c) => c.title === k); + if (col) { + row[k] = await serializeCellValue(context, { + value: v, + column: col, + siteUrl: param.ncSiteUrl, + }); + } + } + } + return { data }; + }; + const baseModel = await Model.getBaseModelSQL(context, { id: model.id, viewId: view?.id, @@ -576,7 +612,7 @@ export class ExportService { try { await this.recursiveRead( context, - formatData, + dataExportMode ? formatAndSerialize : formatData, baseModel, dataStream, model, @@ -585,6 +621,8 @@ export class ExportService { limit, fields, true, + param.delimiter, + dataExportMode, ); } catch (e) { this.debugLog(e); @@ -670,13 +708,13 @@ export class ExportService { linkStream.push(null); } else { - linkStream.push(null); + if (linkStream) linkStream.push(null); } } async recursiveRead( context: NcContext, - formatter: (data: any) => { data: any }, + formatter: (data: any) => { data: any } | Promise<{ data: any }>, baseModel: BaseModelSqlv2, stream: Readable, model: Model, @@ -685,6 +723,8 @@ export class ExportService { limit: number, fields: string, header = false, + delimiter = ',', + dataExportMode = false, ): Promise { return new Promise((resolve, reject) => { this.datasService @@ -693,7 +733,7 @@ export class ExportService { view, query: { limit, offset, fields }, baseModel, - ignoreViewFilterAndSort: true, + ignoreViewFilterAndSort: !dataExportMode, limitOverride: limit, }) .then((result) => { @@ -701,25 +741,57 @@ export class ExportService { if (!header) { stream.push('\r\n'); } - const { data } = formatter(result.list); - stream.push(unparse(data, { header })); - if (result.pageInfo.isLastPage) { - stream.push(null); - resolve(); + + // check if formatter is async + const formatterPromise = formatter(result.list); + if (formatterPromise instanceof Promise) { + formatterPromise.then(({ data }) => { + stream.push(unparse(data, { header, delimiter })); + if (result.pageInfo.isLastPage) { + stream.push(null); + resolve(); + } else { + this.recursiveRead( + context, + formatter, + baseModel, + stream, + model, + view, + offset + limit, + limit, + fields, + false, + delimiter, + dataExportMode, + ) + .then(resolve) + .catch(reject); + } + }); } else { - this.recursiveRead( - context, - formatter, - baseModel, - stream, - model, - view, - offset + limit, - limit, - fields, - ) - .then(resolve) - .catch(reject); + stream.push(unparse(formatterPromise.data, { header })); + if (result.pageInfo.isLastPage) { + stream.push(null); + resolve(); + } else { + this.recursiveRead( + context, + formatter, + baseModel, + stream, + model, + view, + offset + limit, + limit, + fields, + false, + delimiter, + dataExportMode, + ) + .then(resolve) + .catch(reject); + } } } catch (e) { reject(e); diff --git a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.controller.ts b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.controller.ts index 99d7a1985a..0ce45081ac 100644 --- a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.controller.ts @@ -50,6 +50,7 @@ export class SourceCreateController { const job = await this.jobsService.add(JobTypes.SourceCreate, { context, + user: req.user, baseId, source: body, req: { diff --git a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts index dfeaa495c3..e80cf83763 100644 --- a/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts @@ -46,7 +46,5 @@ export class SourceCreateProcessor { } this.debugLog(`job completed for ${job.id}`); - - return createdSource; } } diff --git a/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.controller.ts b/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.controller.ts index f6b4d0e777..c28673ce63 100644 --- a/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.controller.ts +++ b/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.controller.ts @@ -47,6 +47,7 @@ export class SourceDeleteController { const job = await this.jobsService.add(JobTypes.SourceDelete, { context, + user: req.user, sourceId, req: { user: req.user, diff --git a/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.processor.ts b/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.processor.ts index 5cc31e7ca0..fd2b2539ee 100644 --- a/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.processor.ts @@ -22,7 +22,5 @@ export class SourceDeleteProcessor { }); this.debugLog(`job completed for ${job.id}`); - - return true; } } diff --git a/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts deleted file mode 100644 index f044ac1f1b..0000000000 --- a/packages/nocodb/src/modules/jobs/redis/jobs-event.service.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - OnQueueActive, - OnQueueCompleted, - OnQueueFailed, - Processor, -} from '@nestjs/bull'; -import { Job } from 'bull'; -import { EventEmitter2 } from '@nestjs/event-emitter'; -import { Logger } from '@nestjs/common'; -import { JobEvents, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; - -@Processor(JOBS_QUEUE) -export class JobsEventService { - protected logger = new Logger(JobsEventService.name); - - constructor(private eventEmitter: EventEmitter2) {} - - @OnQueueActive() - onActive(job: Job) { - this.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.ACTIVE, - }); - } - - @OnQueueFailed() - onFailed(job: Job, error: Error) { - this.logger.error( - `---- !! JOB FAILED !! ----\nid:${job.id}\nerror:${error.name} (${error.message})\n\nstack: ${error.stack}`, - ); - - this.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.FAILED, - data: { - error: { - message: error?.message, - }, - }, - }); - } - - @OnQueueCompleted() - onCompleted(job: Job, data: any) { - this.eventEmitter.emit(JobEvents.STATUS, { - id: job.id.toString(), - status: JobStatus.COMPLETED, - data: { - result: data, - }, - }); - } -} diff --git a/packages/nocodb/src/modules/jobs/redis/jobs.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts index 0a581f2d0b..3b0cb097f8 100644 --- a/packages/nocodb/src/modules/jobs/redis/jobs.service.ts +++ b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts @@ -4,6 +4,8 @@ import { Queue } from 'bull'; import type { OnModuleInit } from '@nestjs/common'; import { InstanceCommands, JOBS_QUEUE, JobStatus } from '~/interface/Jobs'; import { JobsRedis } from '~/modules/jobs/redis/jobs-redis'; +import { Job } from '~/models'; +import { RootScopes } from '~/utils/globals'; @Injectable() export class JobsService implements OnModuleInit { @@ -51,11 +53,24 @@ export class JobsService implements OnModuleInit { async add(name: string, data: any) { await this.toggleQueue(); - const job = await this.jobsQueue.add(name, data, { + const context = { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + ...(data?.context || {}), + }; + + const jobData = await Job.insert(context, { + job: name, + status: JobStatus.WAITING, + fk_user_id: data?.user?.id, + }); + + await this.jobsQueue.add(name, data, { + jobId: jobData.id, removeOnComplete: true, }); - return job; + return jobData; } async jobStatus(jobId: string) { @@ -74,30 +89,6 @@ export class JobsService implements OnModuleInit { ]); } - async getJobWithData(data: any) { - const jobs = await this.jobsQueue.getJobs([ - // 'completed', - JobStatus.WAITING, - JobStatus.ACTIVE, - JobStatus.DELAYED, - // 'failed', - JobStatus.PAUSED, - ]); - - const job = jobs.find((j) => { - for (const key in data) { - if (j.data[key]) { - if (j.data[key] !== data[key]) return false; - } else { - return false; - } - } - return true; - }); - - return job; - } - async resumeQueue() { this.logger.log('Resuming global queue'); await this.jobsQueue.resume(); diff --git a/packages/nocodb/src/modules/noco.module.ts b/packages/nocodb/src/modules/noco.module.ts index 4cc916c988..7866bd16b5 100644 --- a/packages/nocodb/src/modules/noco.module.ts +++ b/packages/nocodb/src/modules/noco.module.ts @@ -100,6 +100,8 @@ import { CommandPaletteService } from '~/services/command-palette.service'; import { CommandPaletteController } from '~/controllers/command-palette.controller'; import { ExtensionsService } from '~/services/extensions.service'; import { ExtensionsController } from '~/controllers/extensions.controller'; +import { JobsMetaService } from '~/services/jobs-meta.service'; +import { JobsMetaController } from '~/controllers/jobs-meta.controller'; /* Datas */ import { DataTableController } from '~/controllers/data-table.controller'; @@ -178,6 +180,7 @@ export const nocoModuleMetadata = { NotificationsController, CommandPaletteController, ExtensionsController, + JobsMetaController, /* Datas */ DataTableController, @@ -246,6 +249,7 @@ export const nocoModuleMetadata = { NotificationsService, CommandPaletteService, ExtensionsService, + JobsMetaService, /* Datas */ DataTableService, diff --git a/packages/nocodb/src/plugins/s3/S3.ts b/packages/nocodb/src/plugins/s3/S3.ts index 5908fed023..4d40424f40 100644 --- a/packages/nocodb/src/plugins/s3/S3.ts +++ b/packages/nocodb/src/plugins/s3/S3.ts @@ -108,10 +108,13 @@ export default class S3 implements IStorageAdapterV2 { }); } - public async getSignedUrl(key, expiresInSeconds = 7200) { + public async getSignedUrl(key, expiresInSeconds = 7200, filename?: string) { const command = new GetObjectCommand({ Key: key, Bucket: this.input.bucket, + ...(filename + ? { ResponseContentDisposition: `attachment; filename="${filename}" ` } + : {}), }); return getSignedUrl(this.s3Client, command, { expiresIn: expiresInSeconds, diff --git a/packages/nocodb/src/schema/swagger-v2.json b/packages/nocodb/src/schema/swagger-v2.json index b60f29dfe3..ecaed2488d 100644 --- a/packages/nocodb/src/schema/swagger-v2.json +++ b/packages/nocodb/src/schema/swagger-v2.json @@ -11817,6 +11817,96 @@ } ] } + }, + "/api/v2/jobs/{baseId}": { + "post": { + "summary": "Get Jobs", + "operationId": "jobs-list", + "description": "Get list of jobs for a given base for the user", + "tags": [ + "Jobs" + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "job": { + "type": "string" + }, + "status": { + "type": "string" + } + } + } + } + } + } + }, + "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "p124dflkcvasewh", + "type": "string" + }, + "name": "baseId", + "in": "path", + "required": true, + "description": "Unique Base ID" + }, + { + "$ref": "#/components/parameters/xc-auth" + } + ] + }, + "/api/v2/export/{viewId}/{exportAs}": { + "post": { + "summary": "Trigger export as job", + "operationId": "export-data", + "description": "Trigger export as job", + "tags": [ + "Export" + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "vw124dflkcvasewh", + "type": "string" + }, + "name": "viewId", + "in": "path", + "required": true, + "description": "Unique View ID" + }, + { + "schema": { + "type": "string", + "enum": [ + "csv" + ] + }, + "name": "exportAs", + "in": "path", + "required": true, + "description": "Export as format" + }, + { + "$ref": "#/components/parameters/xc-auth" + } + ] } }, "components": { diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index ba546821b3..5437020383 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -17943,14 +17943,57 @@ } ] }, - "/jobs/status": { + "/api/v2/jobs/{baseId}": { "post": { - "summary": "Jobs Status", - "operationId": "jobs-status", - "description": "Get job status", + "summary": "Get Jobs", + "operationId": "jobs-list", + "description": "Get list of jobs for a given base for the user", "tags": [ "Jobs" ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "job": { + "type": "string" + }, + "status": { + "type": "string" + } + } + } + } + } + } + }, + "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "p124dflkcvasewh", + "type": "string" + }, + "name": "baseId", + "in": "path", + "required": true, + "description": "Unique Base ID" + }, + { + "$ref": "#/components/parameters/xc-auth" + } + ] + }, + "/api/v2/export/{viewId}/{exportAs}": { + "post": { + "summary": "Trigger export as job", + "operationId": "export-data", + "description": "Trigger export as job", + "tags": [ + "Export" + ], "requestBody": { "content": { "application/json": { @@ -17962,6 +18005,29 @@ } }, "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "vw124dflkcvasewh", + "type": "string" + }, + "name": "viewId", + "in": "path", + "required": true, + "description": "Unique View ID" + }, + { + "schema": { + "type": "string", + "enum": [ + "csv" + ] + }, + "name": "exportAs", + "in": "path", + "required": true, + "description": "Export as format" + }, { "$ref": "#/components/parameters/xc-auth" } diff --git a/packages/nocodb/src/services/jobs-meta.service.spec.ts b/packages/nocodb/src/services/jobs-meta.service.spec.ts new file mode 100644 index 0000000000..d95546a254 --- /dev/null +++ b/packages/nocodb/src/services/jobs-meta.service.spec.ts @@ -0,0 +1,19 @@ +import { Test } from '@nestjs/testing'; +import { JobsMetaService } from './jobs-meta.service'; +import type { TestingModule } from '@nestjs/testing'; + +describe('JobsMetaService', () => { + let service: JobsMetaService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [JobsMetaService], + }).compile(); + + service = module.get(JobsMetaService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/packages/nocodb/src/services/jobs-meta.service.ts b/packages/nocodb/src/services/jobs-meta.service.ts new file mode 100644 index 0000000000..cada6cef87 --- /dev/null +++ b/packages/nocodb/src/services/jobs-meta.service.ts @@ -0,0 +1,83 @@ +import { Injectable } from '@nestjs/common'; +import dayjs from 'dayjs'; +import type { NcContext, NcRequest } from '~/interface/config'; +import type { JobTypes } from '~/interface/Jobs'; +import { JobStatus } from '~/interface/Jobs'; +import { Job } from '~/models'; +import Noco from '~/Noco'; + +@Injectable() +export class JobsMetaService { + constructor() {} + + async list( + context: NcContext, + param: { job?: JobTypes; status?: JobStatus }, + req: NcRequest, + ) { + /* + * List jobs for the current base. + * If the job is not created by the current user, exclude the result. + * List jobs updated in the last 1 hour or jobs that are still active(, waiting, or delayed). + */ + return Job.list(context, { + xcCondition: { + _and: [ + ...(param.job + ? [ + { + job: { + eq: param.job, + }, + }, + ] + : []), + ...(param.status + ? [ + { + status: { + eq: param.status, + }, + }, + ] + : []), + { + _or: [ + { + updated_at: { + gt: Noco.ncMeta.formatDateTime( + dayjs().subtract(1, 'hour').toISOString(), + ), + }, + }, + { + status: { + eq: JobStatus.ACTIVE, + }, + }, + { + status: { + eq: JobStatus.WAITING, + }, + }, + { + status: { + eq: JobStatus.DELAYED, + }, + }, + ], + }, + ], + }, + }).then((jobs) => { + return jobs.map((job) => { + if (job.fk_user_id === req.user.id) { + return job; + } else { + const { result, ...rest } = job; + return rest; + } + }); + }); + } +} diff --git a/packages/nocodb/src/utils/acl.ts b/packages/nocodb/src/utils/acl.ts index f32253a540..ee98a7b39d 100644 --- a/packages/nocodb/src/utils/acl.ts +++ b/packages/nocodb/src/utils/acl.ts @@ -143,6 +143,9 @@ const permissionScopes = { 'extensionCreate', 'extensionUpdate', 'extensionDelete', + + // Jobs + 'jobList', ], }; @@ -209,6 +212,7 @@ const rolePermissions: extensionList: true, extensionRead: true, + jobList: true, commentList: true, commentsCount: true, auditListRow: true, diff --git a/packages/nocodb/src/utils/globals.ts b/packages/nocodb/src/utils/globals.ts index e4fcc14ad5..e70c7c8ff6 100644 --- a/packages/nocodb/src/utils/globals.ts +++ b/packages/nocodb/src/utils/globals.ts @@ -51,6 +51,7 @@ export enum MetaTable { COMMENTS = 'nc_comments', USER_COMMENTS_NOTIFICATIONS_PREFERENCE = 'nc_user_comment_notifications_preference', COMMENTS_REACTIONS = 'nc_comment_reactions', + JOBS = 'nc_jobs', } export enum MetaTableOldV2 { @@ -171,6 +172,7 @@ export enum CacheScope { DASHBOARD_PROJECT_DB_PROJECT_LINKING = 'dashboardProjectDBProjectLinking', SINGLE_QUERY = 'singleQuery', JOBS = 'nc_jobs', + JOBS_POLLING = 'nc_jobs_polling', PRESIGNED_URL = 'presignedUrl', STORE = 'store', PROJECT_ALIAS = 'baseAlias', @@ -281,6 +283,7 @@ export const RootScopeTables = { MetaTable.PLUGIN, MetaTable.STORE, MetaTable.NOTIFICATION, + MetaTable.JOBS, // Temporarily added need to be discussed within team MetaTable.AUDIT, ],