mirror of https://github.com/nocodb/nocodb
Browse Source
* wip: add job manager Signed-off-by: Pranav C <pranavxc@gmail.com> * wip: job manager impl Signed-off-by: Pranav C <pranavxc@gmail.com> * wip: migrations Signed-off-by: Pranav C <pranavxc@gmail.com> * wip Signed-off-by: Pranav C <pranavxc@gmail.com> * wip: data sync Signed-off-by: Pranav C <pranavxc@gmail.com> * feat: sync source creation Signed-off-by: Pranav C <pranavxc@gmail.com> * feat: api integration Signed-off-by: Pranav C <pranavxc@gmail.com> * chore: remove unused files Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: update to work with project id, handle exceptions Signed-off-by: Pranav C <pranavxc@gmail.com> * feat: show more progress details Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: extract id from source creation api response Signed-off-by: Pranav C <pranavxc@gmail.com> * feat: bring latest changes Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: column creation and data sync Signed-off-by: Pranav C <pranavxc@gmail.com> * refactor: ui improvements Signed-off-by: Pranav C <pranavxc@gmail.com> * refactor: bring changes from sync branch Signed-off-by: Pranav C <pranavxc@gmail.com> * refactor: avoid opening additional socket Signed-off-by: Pranav C <pranavxc@gmail.com> * chore: update package-lock Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: wait until data and LTAR insertion completes Signed-off-by: Pranav C <pranavxc@gmail.com> * feat: load table list after sync completes Signed-off-by: Pranav C <pranavxc@gmail.com> * enhancement: add navigation back to dashboard Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: column order as in base, column visibility clean-up Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * refactor: ui updates Signed-off-by: Pranav C <pranavxc@gmail.com> * enhancement: load first table after sync Signed-off-by: Pranav C <pranavxc@gmail.com> * refactor: common routine to sanitize column names Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * enhancement: accept both hared base id / url Signed-off-by: Pranav C <pranavxc@gmail.com> * refactor: sanitize table name Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * sync: fetch improvements Signed-off-by: mertmit <mertmit99@gmail.com> * sync: fix nestedLookup Signed-off-by: mertmit <mertmit99@gmail.com> * fix: dateTime datatype support Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * enhancement: add validation for credential form Signed-off-by: Pranav C <pranavxc@gmail.com> * fix: migrate default values if configured Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: include support for created time, modified time Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * fix: rollup column function map Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * feat: log column not migrated info Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * feat: fetch template Signed-off-by: mertmit <mertmit99@gmail.com> * fix: move record id & hash to bottom of list Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com> * refactor: add import from airtable under project tabs menu Signed-off-by: Pranav C <pranavxc@gmail.com> Co-authored-by: Raju Udava <86527202+dstala@users.noreply.github.com> Co-authored-by: mertmit <mertmit99@gmail.com>pull/2051/head
Pranav C
3 years ago
committed by
GitHub
56 changed files with 3575 additions and 96 deletions
@ -0,0 +1,281 @@
|
||||
<template> |
||||
<v-dialog v-model="airtableModal" max-width="min(900px, 90%)"> |
||||
<v-card class="nc-import-card"> |
||||
<v-toolbar class="elevation-0" height="68"> |
||||
<h3 class="mt-2 grey--text"> |
||||
{{ $t('title.importFromAirtable') }} : |
||||
<span v-if="step === 1" @dblclick="$set(syncSource.details,'syncViews',true)">Credentials<span |
||||
v-if="syncSource && syncSource.details && syncSource.details.syncViews" |
||||
>.</span></span> |
||||
<span v-else-if="step === 2">Logs</span> |
||||
</h3> |
||||
|
||||
<v-spacer /> |
||||
</v-toolbar> |
||||
|
||||
<div class="h-100" style="width: 100%"> |
||||
<div> |
||||
<v-card v-if="step === 1" class="py-6"> |
||||
<v-form v-model="valid"> |
||||
<div v-if="syncSource" class="px-10 mt-1 mx-auto" style="max-width: 400px"> |
||||
<v-text-field |
||||
v-model="syncSource.details.apiKey" |
||||
outlined |
||||
dense |
||||
label="Api Key" |
||||
class="caption" |
||||
:rules="[v=> !!v || 'Api Key is required']" |
||||
/> |
||||
<v-text-field |
||||
v-model="syncSourceUrlOrId" |
||||
outlined |
||||
dense |
||||
label="Shared Base ID / URL" |
||||
class="caption" |
||||
:rules="[(v) => !!v || 'Shared Base ID / URL is required']" |
||||
/> |
||||
</div> |
||||
</v-form> <v-card-actions class="justify-center pb-6"> |
||||
<v-btn |
||||
v-t="['c:sync-airtable:save-and-sync']" |
||||
:disabled="!valid" |
||||
large |
||||
color="primary" |
||||
@click="saveAndSync" |
||||
> |
||||
Save & Sync |
||||
</v-btn> |
||||
</v-card-actions> |
||||
</v-card> |
||||
|
||||
<v-card |
||||
v-if="step === 2" |
||||
class="py-4 mt-4" |
||||
> |
||||
<v-card |
||||
ref="log" |
||||
dark |
||||
class="mt-2 mx-4 px-2 elevation-0 green--text" |
||||
height="500" |
||||
style="overflow-y: auto" |
||||
> |
||||
<div v-for="({msg , status}, i) in progress" :key="i"> |
||||
<v-icon v-if="status==='FAILED'" color="red" size="15"> |
||||
mdi-close-circle-outline |
||||
</v-icon> |
||||
<v-icon v-else color="green" size="15"> |
||||
mdi-currency-usd |
||||
</v-icon> |
||||
<span class="caption nc-text">{{ msg }}</span> |
||||
</div> |
||||
<div |
||||
v-if="!progress || !progress.length || progress[progress.length-1].status !== 'COMPLETED' && progress[progress.length-1].status !== 'FAILED'" |
||||
class="" |
||||
> |
||||
<v-icon color="green" size="15"> |
||||
mdi-loading mdi-spin |
||||
</v-icon> |
||||
<span class="caption nc-text">Syncing |
||||
</span> |
||||
<!-- <div class="nc-progress" />--> |
||||
</div> |
||||
</v-card> |
||||
|
||||
<div |
||||
v-if="progress && progress.length && progress[progress.length-1].status === 'COMPLETED'" |
||||
class="pa-4 pt-8 text-center" |
||||
> |
||||
<v-btn large color="primary" @click="airtableModal=false"> |
||||
Go to dashboard |
||||
</v-btn> |
||||
</div> |
||||
</v-card> |
||||
</div> |
||||
</div> |
||||
</v-card> |
||||
</v-dialog> |
||||
</template> |
||||
|
||||
<script> |
||||
import io from 'socket.io-client' |
||||
|
||||
export default { |
||||
name: 'ImportFromAirtable', |
||||
props: { |
||||
value: Boolean |
||||
}, |
||||
data: () => ({ |
||||
valid: false, |
||||
socket: null, |
||||
step: 1, |
||||
progress: [], |
||||
syncSource: null, |
||||
syncSourceUrlOrId: '' |
||||
}), |
||||
computed: { |
||||
airtableModal: { |
||||
set(v) { |
||||
this.$emit('input', v) |
||||
}, |
||||
get() { |
||||
return this.value |
||||
} |
||||
} |
||||
}, |
||||
watch: { |
||||
syncSourceUrlOrId(v) { |
||||
if (this.syncSource && this.syncSource.details) { |
||||
const m = v && v.match(/(exp|shr).{14}/g) |
||||
this.syncSource.details.shareId = m ? m[0] : null |
||||
} |
||||
} |
||||
}, |
||||
created() { |
||||
this.socket = io(new URL(this.$axios.defaults.baseURL, window.location.href.split(/[?#]/)[0]).href, { |
||||
extraHeaders: { 'xc-auth': this.$store.state.users.token } |
||||
}) |
||||
this.socket.on('connect_error', () => { |
||||
this.socket.disconnect() |
||||
this.socket = null |
||||
}) |
||||
|
||||
const socket = this.socket |
||||
socket.on('connect', function(data) { |
||||
console.log(socket.id) |
||||
console.log('socket connected', data) |
||||
}) |
||||
|
||||
socket.on('progress', (d) => { |
||||
this.progress.push(d) |
||||
|
||||
this.$nextTick(() => { |
||||
if (this.$refs.log) { |
||||
const el = this.$refs.log.$el |
||||
el.scrollTop = el.scrollHeight |
||||
} |
||||
}) |
||||
|
||||
if (d.status === 'COMPLETED') { |
||||
this.$store.dispatch('project/_loadTables', { |
||||
dbKey: '0.projectJson.envs._noco.db.0', |
||||
key: '0.projectJson.envs._noco.db.0.tables', |
||||
_nodes: { |
||||
dbAlias: 'db', |
||||
env: '_noco', |
||||
type: 'tableDir' |
||||
} |
||||
}).then(() => this.$store.dispatch('tabs/loadFirstTableTab')) |
||||
} |
||||
}) |
||||
this.loadSyncSrc() |
||||
}, |
||||
beforeDestroy() { |
||||
if (this.socket) { |
||||
this.socket.disconnect() |
||||
} |
||||
}, |
||||
methods: { |
||||
async saveAndSync() { |
||||
await this.createOrUpdate() |
||||
this.sync() |
||||
}, |
||||
sync() { |
||||
this.step = 2 |
||||
this.$axios.post(`/api/v1/db/meta/syncs/${this.syncSource.id}/trigger`, this.payload, { |
||||
params: { |
||||
id: this.socket.id |
||||
} |
||||
}) |
||||
}, |
||||
async loadSyncSrc() { |
||||
const { data: { list: srcs } } = await this.$axios.get(`/api/v1/db/meta/projects/${this.projectId}/syncs`) |
||||
if (srcs && srcs[0]) { |
||||
srcs[0].details = srcs[0].details || {} |
||||
this.syncSource = srcs[0] |
||||
this.syncSourceUrlOrId = srcs[0].details.shareId |
||||
} else { |
||||
this.syncSource = { |
||||
type: 'Airtable', |
||||
details: { |
||||
syncInterval: '15mins', |
||||
syncDirection: 'Airtable to NocoDB', |
||||
syncRetryCount: 1, |
||||
|
||||
syncViews: false, |
||||
|
||||
apiKey: '', |
||||
shareId: '' |
||||
} |
||||
} |
||||
} |
||||
}, |
||||
async createOrUpdate() { |
||||
try { |
||||
const { id, ...payload } = this.syncSource |
||||
if (id) { |
||||
await this.$axios.patch(`/api/v1/db/meta/syncs/${id}`, payload) |
||||
} else { |
||||
this.syncSource = (await this.$axios.post(`/api/v1/db/meta/projects/${this.projectId}/syncs`, payload)).data |
||||
} |
||||
} catch (e) { |
||||
this.$toast.error(await this._extractSdkResponseErrorMsg(e)).goAway(3000) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
</script> |
||||
|
||||
<style scoped> |
||||
|
||||
.nc-progress { |
||||
margin-left: 12px; |
||||
position: relative; |
||||
width: 5px; |
||||
height: 5px; |
||||
border-radius: 5px; |
||||
background-color: #9880ff; |
||||
color: #9880ff; |
||||
animation: dotFlashing 1s infinite linear alternate; |
||||
animation-delay: .5s; |
||||
} |
||||
|
||||
.nc-progress::before, .nc-progress::after { |
||||
content: ''; |
||||
display: inline-block; |
||||
position: absolute; |
||||
top: 0; |
||||
} |
||||
|
||||
.nc-progress::before { |
||||
left: -7.5px; |
||||
width: 5px; |
||||
height: 5px; |
||||
border-radius: 5px; |
||||
background-color: #9880ff; |
||||
color: #9880ff; |
||||
animation: dotFlashing 1s infinite alternate; |
||||
animation-delay: 0s; |
||||
} |
||||
|
||||
.nc-progress::after { |
||||
left: 7.5px; |
||||
width: 5px; |
||||
height: 5px; |
||||
border-radius: 5px; |
||||
background-color: var(--v-primary-base); |
||||
color: var(--v-primary-base); |
||||
animation: dotFlashing 1s infinite alternate; |
||||
animation-delay: 1s; |
||||
} |
||||
|
||||
@keyframes dotFlashing { |
||||
0% { |
||||
background-color: var(--v-primary-base); |
||||
} |
||||
50%, |
||||
100% { |
||||
background-color: var(--v-backgroundColor-base); |
||||
} |
||||
} |
||||
|
||||
</style> |
||||
@ -0,0 +1,35 @@
|
||||
import JobsMgr from './JobsMgr'; |
||||
import Emittery from 'emittery'; |
||||
|
||||
export default class EmitteryJobsMgr extends JobsMgr { |
||||
emitter: Emittery; |
||||
|
||||
constructor() { |
||||
super(); |
||||
this.emitter = new Emittery(); |
||||
} |
||||
|
||||
add(jobName: string, payload: any): Promise<any> { |
||||
return this.emitter.emit(jobName, payload); |
||||
} |
||||
|
||||
addJobWorker( |
||||
jobName: string, |
||||
workerFn: ( |
||||
payload: any, |
||||
progressCbk?: (payload: any, msg?: string) => void |
||||
) => void |
||||
) { |
||||
this.emitter.on(jobName, async payload => { |
||||
try { |
||||
await workerFn(payload, msg => |
||||
this.invokeProgressCbks(jobName, payload, msg) |
||||
); |
||||
await this.invokeSuccessCbks(jobName, payload); |
||||
} catch (e) { |
||||
console.log(e); |
||||
await this.invokeFailureCbks(jobName, payload, e); |
||||
} |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,65 @@
|
||||
export default abstract class JobsMgr { |
||||
protected successCbks: Array<{ |
||||
[jobName: string]: (payload: any) => void; |
||||
}> = []; |
||||
protected failureCbks: Array<{ |
||||
[jobName: string]: (payload: any, error: Error) => void; |
||||
}> = []; |
||||
protected progressCbks: Array<{ |
||||
[jobName: string]: (payload: any, msg?: string) => void; |
||||
}> = []; |
||||
|
||||
public abstract add<T>(jobName: string, payload: T): Promise<any>; |
||||
|
||||
public abstract addJobWorker( |
||||
jobName: string, |
||||
workerFn: ( |
||||
payload: any, |
||||
progressCbk?: (payload: any, msg?: string) => void |
||||
) => void, |
||||
options?: { |
||||
onSuccess?: (payload: any) => void; |
||||
onFailure?: (payload: any, msg: string) => void; |
||||
onProgress?: (payload: any, msgOrData: any) => void; |
||||
} |
||||
); |
||||
|
||||
addSuccessCbk(jobName: string, cbk: (payload: any) => void) { |
||||
this.successCbks[jobName] = this.successCbks[jobName] || []; |
||||
this.successCbks[jobName].push(cbk); |
||||
} |
||||
|
||||
addFailureCbk(jobName: string, cbk: (payload: any, msg: string) => void) { |
||||
this.failureCbks[jobName] = this.failureCbks[jobName] || []; |
||||
this.failureCbks[jobName].push(cbk); |
||||
} |
||||
addProgressCbk( |
||||
jobName: string, |
||||
cbk: (payload: any, progress: string) => void |
||||
) { |
||||
this.progressCbks[jobName] = this.progressCbks[jobName] || []; |
||||
this.progressCbks[jobName].push(cbk); |
||||
} |
||||
|
||||
protected async invokeSuccessCbks(jobName: string, payload: any) { |
||||
await Promise.all(this.successCbks?.[jobName]?.map(cb => cb(payload))); |
||||
} |
||||
protected async invokeFailureCbks( |
||||
jobName: string, |
||||
payload: any, |
||||
error?: Error |
||||
) { |
||||
await Promise.all( |
||||
this.failureCbks?.[jobName]?.map(cb => cb(payload, error)) |
||||
); |
||||
} |
||||
protected async invokeProgressCbks( |
||||
jobName: string, |
||||
payload: any, |
||||
msg?: string |
||||
) { |
||||
await Promise.all( |
||||
this.progressCbks?.[jobName]?.map(cb => cb(payload, msg)) |
||||
); |
||||
} |
||||
} |
||||
@ -0,0 +1,20 @@
|
||||
import JobsMgr from './JobsMgr'; |
||||
import EmitteryJobsMgr from './EmitteryJobsMgr'; |
||||
import RedisJobsMgr from './RedisJobsMgr'; |
||||
|
||||
export default class NocoJobs { |
||||
private static client: JobsMgr; |
||||
|
||||
private static init() { |
||||
if (process.env.NC_REDIS_URL) { |
||||
this.client = new RedisJobsMgr(process.env.NC_REDIS_URL); |
||||
} else { |
||||
this.client = new EmitteryJobsMgr(); |
||||
} |
||||
} |
||||
|
||||
public static get jobsMgr(): JobsMgr { |
||||
if (!this.client) this.init(); |
||||
return this.client; |
||||
} |
||||
} |
||||
@ -0,0 +1,54 @@
|
||||
import { Queue, Worker } from 'bullmq'; |
||||
import Redis from 'ioredis'; |
||||
import JobsMgr from './JobsMgr'; |
||||
|
||||
export default class RedisJobsMgr extends JobsMgr { |
||||
queue: { [jobName: string]: Queue }; |
||||
workers: { [jobName: string]: Worker }; |
||||
connection: Redis; |
||||
|
||||
constructor(config: any) { |
||||
super(); |
||||
this.queue = {}; |
||||
this.workers = {}; |
||||
this.connection = new Redis(config); |
||||
} |
||||
|
||||
async add( |
||||
jobName: string, |
||||
payload: any |
||||
// options?: {
|
||||
// onSuccess?: (payload: any) => void;
|
||||
// onFailure?: (payload: any, msg: string) => void;
|
||||
// onProgress?: (payload: any, msgOrData: any) => void;
|
||||
// }
|
||||
): Promise<any> { |
||||
this.queue[jobName] = |
||||
this.queue[jobName] || |
||||
new Queue(jobName, { connection: this.connection }); |
||||
this.queue[jobName].add(jobName, payload); |
||||
} |
||||
|
||||
addJobWorker( |
||||
jobName: string, |
||||
workerFn: ( |
||||
payload: any, |
||||
progressCbk?: (payload: any, msg?: string) => void |
||||
) => void |
||||
) { |
||||
this.workers[jobName] = new Worker( |
||||
jobName, |
||||
async payload => { |
||||
try { |
||||
await workerFn(payload.data, (...args) => |
||||
this.invokeProgressCbks(jobName, ...args) |
||||
); |
||||
await this.invokeFailureCbks(jobName, payload.data); |
||||
} catch (e) { |
||||
await this.invokeFailureCbks(jobName, payload.data); |
||||
} |
||||
}, |
||||
{ connection: this.connection } |
||||
); |
||||
} |
||||
} |
||||
@ -0,0 +1,57 @@
|
||||
import Noco from '../noco/Noco'; |
||||
import { MetaTable } from '../utils/globals'; |
||||
|
||||
export default class SyncLogs { |
||||
id?: string; |
||||
project_id?: string; |
||||
fk_sync_source_id?: string; |
||||
time_taken?: string; |
||||
status?: string; |
||||
status_details?: string; |
||||
|
||||
constructor(syncLog: Partial<SyncLogs>) { |
||||
Object.assign(this, syncLog); |
||||
} |
||||
|
||||
static async list(projectId: string, ncMeta = Noco.ncMeta) { |
||||
const syncLogs = await ncMeta.metaList(null, null, MetaTable.SYNC_LOGS, { |
||||
condition: { |
||||
project_id: projectId |
||||
}, |
||||
orderBy: { |
||||
created_at: 'asc' |
||||
} |
||||
}); |
||||
return syncLogs?.map(h => new SyncLogs(h)); |
||||
} |
||||
|
||||
public static async insert( |
||||
syncLog: Partial< |
||||
SyncLogs & { |
||||
created_at?; |
||||
updated_at?; |
||||
} |
||||
>, |
||||
ncMeta = Noco.ncMeta |
||||
) { |
||||
const insertObj = { |
||||
project_id: syncLog?.project_id, |
||||
fk_sync_source_id: syncLog?.fk_sync_source_id, |
||||
time_taken: syncLog?.time_taken, |
||||
status: syncLog?.status, |
||||
status_details: syncLog?.status_details |
||||
}; |
||||
|
||||
const { id } = await ncMeta.metaInsert2( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_LOGS, |
||||
insertObj |
||||
); |
||||
return new SyncLogs({ ...insertObj, id }); |
||||
} |
||||
|
||||
static async delete(syncLogId: any, ncMeta = Noco.ncMeta) { |
||||
return await ncMeta.metaDelete(null, null, MetaTable.SYNC_LOGS, syncLogId); |
||||
} |
||||
} |
||||
@ -0,0 +1,135 @@
|
||||
import Noco from '../noco/Noco'; |
||||
import { MetaTable } from '../utils/globals'; |
||||
import extractProps from '../noco/meta/helpers/extractProps'; |
||||
import User from './User'; |
||||
|
||||
export default class SyncSource { |
||||
id?: string; |
||||
title?: string; |
||||
type?: string; |
||||
details?: any; |
||||
deleted?: boolean; |
||||
order?: number; |
||||
project_id?: string; |
||||
fk_user_id?: string; |
||||
|
||||
constructor(syncSource: Partial<SyncSource>) { |
||||
Object.assign(this, syncSource); |
||||
} |
||||
|
||||
public getUser(ncMeta = Noco.ncMeta) { |
||||
return User.get(this.fk_user_id, ncMeta); |
||||
} |
||||
|
||||
public static async get(syncSourceId: string, ncMeta = Noco.ncMeta) { |
||||
const syncSource = await ncMeta.metaGet2( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_SOURCE, |
||||
syncSourceId |
||||
); |
||||
if (syncSource.details && typeof syncSource.details === 'string') { |
||||
try { |
||||
syncSource.details = JSON.parse(syncSource.details); |
||||
} catch {} |
||||
} |
||||
return syncSource && new SyncSource(syncSource); |
||||
} |
||||
|
||||
static async list(projectId: string, ncMeta = Noco.ncMeta) { |
||||
const syncSources = await ncMeta.metaList( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_SOURCE, |
||||
{ |
||||
condition: { |
||||
project_id: projectId |
||||
}, |
||||
orderBy: { |
||||
created_at: 'asc' |
||||
} |
||||
} |
||||
); |
||||
|
||||
for (const syncSource of syncSources) { |
||||
if (syncSource.details && typeof syncSource.details === 'string') { |
||||
try { |
||||
syncSource.details = JSON.parse(syncSource.details); |
||||
} catch {} |
||||
} |
||||
} |
||||
return syncSources?.map(h => new SyncSource(h)); |
||||
} |
||||
|
||||
public static async insert( |
||||
syncSource: Partial< |
||||
SyncSource & { |
||||
created_at?; |
||||
updated_at?; |
||||
} |
||||
>, |
||||
ncMeta = Noco.ncMeta |
||||
) { |
||||
const insertObj = { |
||||
id: syncSource?.id, |
||||
title: syncSource?.title, |
||||
type: syncSource?.type, |
||||
details: syncSource?.details, |
||||
project_id: syncSource?.project_id, |
||||
fk_user_id: syncSource?.fk_user_id |
||||
}; |
||||
|
||||
if (insertObj.details && typeof insertObj.details === 'object') { |
||||
insertObj.details = JSON.stringify(insertObj.details); |
||||
} |
||||
|
||||
const { id } = await ncMeta.metaInsert2( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_SOURCE, |
||||
insertObj |
||||
); |
||||
|
||||
return this.get(id, ncMeta); |
||||
} |
||||
|
||||
public static async update( |
||||
syncSourceId: string, |
||||
syncSource: Partial<SyncSource>, |
||||
ncMeta = Noco.ncMeta |
||||
) { |
||||
const updateObj = extractProps(syncSource, [ |
||||
'id', |
||||
'title', |
||||
'type', |
||||
'details', |
||||
'deleted', |
||||
'order', |
||||
'project_id' |
||||
]); |
||||
|
||||
if (updateObj.details && typeof updateObj.details === 'object') { |
||||
updateObj.details = JSON.stringify(updateObj.details); |
||||
} |
||||
|
||||
// set meta
|
||||
await ncMeta.metaUpdate( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_SOURCE, |
||||
updateObj, |
||||
syncSourceId |
||||
); |
||||
|
||||
return this.get(syncSourceId, ncMeta); |
||||
} |
||||
|
||||
static async delete(syncSourceId: any, ncMeta = Noco.ncMeta) { |
||||
return await ncMeta.metaDelete( |
||||
null, |
||||
null, |
||||
MetaTable.SYNC_SOURCE, |
||||
syncSourceId |
||||
); |
||||
} |
||||
} |
||||
@ -0,0 +1,6 @@
|
||||
export abstract class NocoSyncSourceAdapter { |
||||
public abstract init(): Promise<void>; |
||||
public abstract destProjectWrite(): Promise<any>; |
||||
public abstract destSchemaWrite(): Promise<any>; |
||||
public abstract destDataWrite(): Promise<any>; |
||||
} |
||||
@ -0,0 +1,7 @@
|
||||
export abstract class NocoSyncSourceAdapter { |
||||
public abstract init(): Promise<void>; |
||||
public abstract srcSchemaGet(): Promise<any>; |
||||
public abstract srcDataLoad(): Promise<any>; |
||||
public abstract srcDataListen(): Promise<any>; |
||||
public abstract srcDataPoll(): Promise<any>; |
||||
} |
||||
@ -0,0 +1,170 @@
|
||||
const axios = require('axios').default; |
||||
|
||||
var info : any = { |
||||
initialized: false |
||||
}; |
||||
|
||||
async function initialize(shareId) { |
||||
info.cookie = ""; |
||||
let url = `https://airtable.com/${shareId}` |
||||
|
||||
try { |
||||
const hreq = await axios.get(url, { |
||||
"headers": { |
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", |
||||
"accept-language": "en-US,en;q=0.9", |
||||
"sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", |
||||
"sec-ch-ua-mobile": "?0", |
||||
"sec-ch-ua-platform": "\"Linux\"", |
||||
"sec-fetch-dest": "document", |
||||
"sec-fetch-mode": "navigate", |
||||
"sec-fetch-site": "none", |
||||
"sec-fetch-user": "?1", |
||||
"upgrade-insecure-requests": "1", |
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36" |
||||
}, |
||||
"referrerPolicy": "strict-origin-when-cross-origin", |
||||
"body": null, |
||||
"method": "GET" |
||||
}) |
||||
.then(response => { |
||||
for(const ck of response.headers['set-cookie']) { |
||||
info.cookie += ck.split(';')[0] + '; ' |
||||
} |
||||
return response.data |
||||
}) |
||||
|
||||
info.headers = JSON.parse(hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim()) |
||||
info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()) |
||||
info.baseInfo = decodeURIComponent(info.link).match(/{(.*)}/g)[0].split('&').reduce((result, el) => { |
||||
try { |
||||
return Object.assign(result, JSON.parse((el.includes('=')?el.split('=')[1]:el))) |
||||
} catch(e) { |
||||
if(el.includes('=')) { |
||||
return Object.assign(result, { [el.split('=')[0]]: el.split('=')[1]}) |
||||
} |
||||
} |
||||
}, {}) |
||||
info.baseId = info.baseInfo.applicationId; |
||||
info.initialized = true; |
||||
} catch (e) { |
||||
console.log(e); |
||||
info.initialized = false; |
||||
} |
||||
} |
||||
|
||||
async function read() { |
||||
if (info.initialized) { |
||||
const resreq = await axios("https://airtable.com" + info.link, { |
||||
"headers": { |
||||
"accept": "*/*", |
||||
"accept-language": "en-US,en;q=0.9", |
||||
"sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", |
||||
"sec-ch-ua-mobile": "?0", |
||||
"sec-ch-ua-platform": "\"Linux\"", |
||||
"sec-fetch-dest": "empty", |
||||
"sec-fetch-mode": "cors", |
||||
"sec-fetch-site": "same-origin", |
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", |
||||
"x-time-zone": "Europe/Berlin", |
||||
"cookie": info.cookie, |
||||
...info.headers |
||||
}, |
||||
"referrerPolicy": "no-referrer", |
||||
"body": null, |
||||
"method": "GET" |
||||
}) |
||||
.then(response => { |
||||
return response.data |
||||
}) |
||||
.catch(() => { |
||||
throw "Error while fetching" |
||||
}) |
||||
|
||||
return {schema: resreq.data, baseId: info.baseId, baseInfo: info.baseInfo} |
||||
} else { |
||||
throw "Please initialize first!" |
||||
} |
||||
} |
||||
|
||||
async function readView(viewId) { |
||||
if (info.initialized) { |
||||
const resreq = await axios(`https://airtable.com/v0.3/view/${viewId}/readData?` + `stringifiedObjectParams=${ |
||||
encodeURIComponent('{}')}&requestId=${info.baseInfo.requestId}&accessPolicy=${encodeURIComponent(JSON.stringify({allowedActions: info.baseInfo.allowedActions, shareId: info.baseInfo.shareId, applicationId: info.baseInfo.applicationId, generationNumber: info.baseInfo.generationNumber, expires: info.baseInfo.expires, signature: info.baseInfo.signature}))}`, {
|
||||
"headers": { |
||||
"accept": "*/*", |
||||
"accept-language": "en-US,en;q=0.9", |
||||
"sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", |
||||
"sec-ch-ua-mobile": "?0", |
||||
"sec-ch-ua-platform": "\"Linux\"", |
||||
"sec-fetch-dest": "empty", |
||||
"sec-fetch-mode": "cors", |
||||
"sec-fetch-site": "same-origin", |
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", |
||||
"x-time-zone": "Europe/Berlin", |
||||
"cookie": info.cookie, |
||||
...info.headers |
||||
}, |
||||
"referrerPolicy": "no-referrer", |
||||
"body": null, |
||||
"method": "GET" |
||||
}) |
||||
.then(response => { |
||||
return response.data |
||||
}) |
||||
.catch(() => { |
||||
throw "Error while fetching" |
||||
}) |
||||
return {view: resreq.data} |
||||
} else { |
||||
throw "Please initialize first!" |
||||
} |
||||
} |
||||
|
||||
async function readTemplate(templateId) { |
||||
if (!info.initialized) { |
||||
await initialize('shrO8aYf3ybwSdDKn'); |
||||
} |
||||
const resreq = await axios(`https://www.airtable.com/v0.3/exploreApplications/${templateId}`, { |
||||
"headers": { |
||||
"accept": "*/*", |
||||
"accept-language": "en-US,en;q=0.9", |
||||
"sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"100\", \"Google Chrome\";v=\"100\"", |
||||
"sec-ch-ua-mobile": "?0", |
||||
"sec-ch-ua-platform": "\"Linux\"", |
||||
"sec-fetch-dest": "empty", |
||||
"sec-fetch-mode": "cors", |
||||
"sec-fetch-site": "same-origin", |
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36", |
||||
"x-time-zone": "Europe/Berlin", |
||||
"cookie": info.cookie, |
||||
...info.headers |
||||
}, |
||||
"referrer": "https://www.airtable.com/", |
||||
"referrerPolicy": "same-origin", |
||||
"body": null, |
||||
"method": "GET", |
||||
"mode": "cors", |
||||
"credentials": "include" |
||||
}) |
||||
.then(response => { |
||||
return response.data |
||||
}) |
||||
.catch(() => { |
||||
throw "Error while fetching" |
||||
}) |
||||
return {template: resreq} |
||||
} |
||||
|
||||
function unicodeToChar(text) { |
||||
return text.replace(/\\u[\dA-F]{4}/gi, function(match) { |
||||
return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); |
||||
}); |
||||
} |
||||
|
||||
export default { |
||||
initialize, |
||||
read, |
||||
readView, |
||||
readTemplate |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,31 @@
|
||||
export const mapTbl = {}; |
||||
|
||||
// static mapping records between aTblId && ncId
|
||||
export const addToMappingTbl = function addToMappingTbl( |
||||
aTblId, |
||||
ncId, |
||||
ncName, |
||||
parent? |
||||
) { |
||||
mapTbl[aTblId] = { |
||||
ncId: ncId, |
||||
ncParent: parent, |
||||
// name added to assist in quick debug
|
||||
ncName: ncName |
||||
}; |
||||
}; |
||||
|
||||
// get NcID from airtable ID
|
||||
export const getNcIdFromAtId = function getNcIdFromAtId(aId) { |
||||
return mapTbl[aId]?.ncId; |
||||
}; |
||||
|
||||
// get nc Parent from airtable ID
|
||||
export const getNcParentFromAtId = function getNcParentFromAtId(aId) { |
||||
return mapTbl[aId]?.ncParent; |
||||
}; |
||||
|
||||
// get nc-title from airtable ID
|
||||
export const getNcNameFromAtId = function getNcNameFromAtId(aId) { |
||||
return mapTbl[aId]?.ncName; |
||||
}; |
||||
@ -0,0 +1,79 @@
|
||||
import { Request, Router } from 'express'; |
||||
// import { Queue } from 'bullmq';
|
||||
// import axios from 'axios';
|
||||
import catchError from '../../helpers/catchError'; |
||||
import { Socket } from 'socket.io'; |
||||
import NocoJobs from '../../../../noco-jobs/NocoJobs'; |
||||
import job, { AirtableSyncConfig } from './helpers/job'; |
||||
import SyncSource from '../../../../noco-models/SyncSource'; |
||||
import Noco from '../../../Noco'; |
||||
import * as jwt from 'jsonwebtoken'; |
||||
const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; |
||||
|
||||
enum SyncStatus { |
||||
PROGRESS = 'PROGRESS', |
||||
COMPLETED = 'COMPLETED', |
||||
FAILED = 'FAILED' |
||||
} |
||||
|
||||
export default (router: Router, clients: { [id: string]: Socket }) => { |
||||
NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, job); |
||||
NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { |
||||
clients?.[payload?.id]?.emit('progress', { |
||||
msg: progress, |
||||
status: SyncStatus.PROGRESS |
||||
}); |
||||
}); |
||||
NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, payload => { |
||||
clients?.[payload?.id]?.emit('progress', { |
||||
msg: 'completed', |
||||
status: SyncStatus.COMPLETED |
||||
}); |
||||
}); |
||||
NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { |
||||
clients?.[payload?.id]?.emit('progress', { |
||||
msg: error?.message || 'Failed due to some internal error', |
||||
status: SyncStatus.FAILED |
||||
}); |
||||
}); |
||||
|
||||
router.post( |
||||
'/api/v1/db/meta/import/airtable', |
||||
catchError((req, res) => { |
||||
NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { |
||||
id: req.query.id, |
||||
...req.body |
||||
}); |
||||
res.json({}); |
||||
}) |
||||
); |
||||
router.post( |
||||
'/api/v1/db/meta/syncs/:syncId/trigger', |
||||
catchError(async (req: Request, res) => { |
||||
const syncSource = await SyncSource.get(req.params.syncId); |
||||
|
||||
const user = await syncSource.getUser(); |
||||
const token = jwt.sign( |
||||
{ |
||||
email: user.email, |
||||
firstname: user.firstname, |
||||
lastname: user.lastname, |
||||
id: user.id, |
||||
roles: user.roles |
||||
}, |
||||
|
||||
Noco.getConfig().auth.jwt.secret, |
||||
Noco.getConfig().auth.jwt.options |
||||
); |
||||
|
||||
NocoJobs.jobsMgr.add<AirtableSyncConfig>(AIRTABLE_IMPORT_JOB, { |
||||
id: req.query.id, |
||||
...(syncSource?.details || {}), |
||||
projectId: syncSource.project_id, |
||||
authToken: token, |
||||
baseURL: (req as any).ncSiteUrl |
||||
}); |
||||
res.json({}); |
||||
}) |
||||
); |
||||
}; |
||||
@ -0,0 +1,53 @@
|
||||
import { Request, Response, Router } from 'express'; |
||||
|
||||
import { Tele } from 'nc-help'; |
||||
import SyncSource from '../../../../noco-models/SyncSource'; |
||||
import { PagedResponseImpl } from '../../helpers/PagedResponse'; |
||||
import ncMetaAclMw from '../../helpers/ncMetaAclMw'; |
||||
|
||||
export async function syncSourceList(req: Request, res: Response) { |
||||
// todo: pagination
|
||||
res.json(new PagedResponseImpl(await SyncSource.list(req.params.projectId))); |
||||
} |
||||
|
||||
export async function syncCreate(req: Request, res: Response) { |
||||
Tele.emit('evt', { evt_type: 'webhooks:created' }); |
||||
const sync = await SyncSource.insert({ |
||||
...req.body, |
||||
fk_user_id: (req as any).user.id, |
||||
project_id: req.params.projectId |
||||
}); |
||||
res.json(sync); |
||||
} |
||||
|
||||
export async function syncDelete(req: Request, res: Response<any>) { |
||||
Tele.emit('evt', { evt_type: 'webhooks:deleted' }); |
||||
res.json(await SyncSource.delete(req.params.syncId)); |
||||
} |
||||
|
||||
export async function syncUpdate(req: Request, res: Response) { |
||||
Tele.emit('evt', { evt_type: 'webhooks:updated' }); |
||||
|
||||
res.json(await SyncSource.update(req.params.syncId, req.body)); |
||||
} |
||||
|
||||
const router = Router({ mergeParams: true }); |
||||
|
||||
router.get( |
||||
'/api/v1/db/meta/projects/:projectId/syncs', |
||||
ncMetaAclMw(syncSourceList, 'syncSourceList') |
||||
); |
||||
router.post( |
||||
'/api/v1/db/meta/projects/:projectId/syncs', |
||||
ncMetaAclMw(syncCreate, 'syncSourceCreate') |
||||
); |
||||
router.delete( |
||||
'/api/v1/db/meta/syncs/:syncId', |
||||
ncMetaAclMw(syncDelete, 'syncSourceDelete') |
||||
); |
||||
router.patch( |
||||
'/api/v1/db/meta/syncs/:syncId', |
||||
ncMetaAclMw(syncUpdate, 'syncSourceUpdate') |
||||
); |
||||
|
||||
export default router; |
||||
@ -0,0 +1,72 @@
|
||||
import Knex from 'knex'; |
||||
import { MetaTable } from '../../utils/globals'; |
||||
|
||||
const up = async (knex: Knex) => { |
||||
await knex.schema.createTable(MetaTable.SYNC_SOURCE, table => { |
||||
table |
||||
.string('id', 20) |
||||
.primary() |
||||
.notNullable(); |
||||
|
||||
table.string('title'); |
||||
table.string('type'); |
||||
table.text('details'); |
||||
table.boolean('deleted'); |
||||
table.boolean('enabled').defaultTo(true); |
||||
table.float('order'); |
||||
|
||||
table.string('project_id', 128); |
||||
table.foreign('project_id').references(`${MetaTable.PROJECT}.id`); |
||||
table.string('fk_user_id', 128); |
||||
table.foreign('fk_user_id').references(`${MetaTable.USERS}.id`); |
||||
|
||||
table.timestamps(true, true); |
||||
}); |
||||
|
||||
await knex.schema.createTable(MetaTable.SYNC_LOGS, table => { |
||||
table |
||||
.string('id', 20) |
||||
.primary() |
||||
.notNullable(); |
||||
|
||||
table.string('project_id', 128); |
||||
table.string('fk_sync_source_id', 20); |
||||
// table
|
||||
// .foreign('fk_sync_source_id')
|
||||
// .references(`${MetaTable.SYNC_SOURCE}.id`);
|
||||
|
||||
table.integer('time_taken'); |
||||
table.string('status'); |
||||
table.text('status_details'); |
||||
|
||||
table.timestamps(true, true); |
||||
}); |
||||
}; |
||||
|
||||
const down = async knex => { |
||||
await knex.schema.dropTable(MetaTable.SYNC_SOURCE); |
||||
}; |
||||
|
||||
export { up, down }; |
||||
|
||||
/** |
||||
* @copyright Copyright (c) 2022, Xgene Cloud Ltd |
||||
* |
||||
* @author Wing-Kam Wong <wingkwong.code@gmail.com> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
Loading…
Reference in new issue