base files

Signed-off-by: Raju Udava <86527202+dstala@users.noreply.github.com>
3 years ago · f1159747df
2 changed files with 375 additions and 0 deletions
--- a/packages/nocodb/tests/export-import/exportSchema.js
+++ b/packages/nocodb/tests/export-import/exportSchema.js
@ -0,0 +1,202 @@
+const Api = require('nocodb-sdk').Api;
+const { UITypes } = require('nocodb-sdk');
+const jsonfile = require("jsonfile");
+
+let ncMap = {}
+let tblSchema = []
+let api = {}
+let viewStore = {}
+
+const ncConfig = {
+  projectName: "x",
+  baseURL: "http://localhost:8080",
+  headers: {
+    'xc-auth': "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfaGJ1aDFmMTNmemc4dTEiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTMwNTU5MzR9.nADVbCbSE0WEbPrpKuq_dlMHrrxieQurYPiOIU2Gf4k"
+  }
+}
+
+// helper routines
+// remove objects containing 0/ false/ null
+// fixme: how to handle when cdf (default value) is configured as 0/ null/ false
+function removeEmpty(obj) {
+  return Object.fromEntries(
+    Object.entries(obj)
+      .filter(([_, v]) => ((v != null) && (v != 0) && (v != false)))
+      .map(([k, v]) => [k, v === Object(v) ? removeEmpty(v) : v])
+  );
+}
+
+// let linksAdded = []
+// function isLinkExists(pId, cId) {
+//   let idx = linksAdded.findIndex(a => a.child === pId && a.parent === cId)
+//   if(idx === -1) {
+//     linksAdded.push({ child: cId, parent: pId })
+//     return false;
+//   }
+//   return true;
+// }
+
+function ncGetColData(cId) {
+
+}
+
+function addColumnSpecificData(c) {
+  // pick required fields to proceed further
+  let col = removeEmpty((({ id, title, column_name, uidt, dt, pk, pv, rqd, dtxp, system }) =>
+    ({ id, title, column_name, uidt, dt, pk, pv, rqd, dtxp, system }))(c))
+
+  let colOptions = null
+  switch(c.uidt) {
+    case UITypes.Formula:
+      colOptions = {
+        formula: c.colOptions.formula,
+        formula_raw: c.colOptions.formula_raw
+      }
+      break
+    case UITypes.LinkToAnotherRecord:
+      colOptions = {
+        fk_model_id: c.fk_model_id,
+        fk_related_model_id: c.colOptions.fk_related_model_id,
+        fk_child_column_id: c.colOptions.fk_child_column_id,
+        fk_parent_column_id: c.colOptions.fk_parent_column_id,
+        type: c.colOptions.type
+      }
+      break;
+    case UITypes.Lookup:
+      colOptions = {
+        fk_model_id: c.fk_model_id,
+        fk_relation_column_id: c.colOptions.fk_relation_column_id,
+        fk_lookup_column_id: c.colOptions.fk_lookup_column_id,
+      }
+      break;
+    case UITypes.Rollup:
+      colOptions = {
+        fk_model_id: c.fk_model_id,
+        fk_relation_column_id: c.colOptions.fk_relation_column_id,
+        fk_rollup_column_id: c.colOptions.fk_rollup_column_id,
+        rollup_function: c.colOptions.rollup_function
+      }
+      break;
+  }
+
+  // colOptions not required for formula
+  if (c.uidt === UITypes.Formula) {
+    col.formula = c.colOptions.formula;
+    col.formula_raw = c.colOptions.formula_raw;
+  }
+  else if(colOptions) col[`colOptions`] = colOptions;
+  return col;
+}
+
+function addViewDetails(v) {
+  // pick required fields to proceed further
+  let view = (({ id, title, type, show_system_fields, lock_type, order }) => (
+    { id, title, type, show_system_fields, lock_type, order }))(v);
+
+  // form view
+  if(v.type === 1) {
+    view.property = (({ heading, subheading, success_msg, redirect_after_secs, email, submit_another_form, show_blank_form }) => (
+      { heading, subheading, success_msg, redirect_after_secs, email, submit_another_form, show_blank_form }))(v.view);
+  }
+
+  // gallery view
+  else if (v.type === 2) {
+    view.property = {fk_cover_image_col_id: ncMap[v.view.fk_cover_image_col_id]}
+  }
+
+  // gallery view doesn't share column information in api yet
+  if(v.type !== 2) {
+    view.columns = viewStore[v.id].map(a => (({ id, width, order, show }) => (
+      { id, width, order, show }))(a))
+    for (let i = 0; i < view.columns?.length; i++)
+      view.columns[i].title = ncMap[viewStore[v.id][i].id]
+
+    view.columns = view.columns.filter(a => a.title.includes('_nc_m2m_') === false)
+  }
+  return view;
+}
+
+// view data stored as is for quick access
+async function storeViewDetails(tableId) {
+  // read view data for each table
+  let viewList = await api.dbView.list(tableId)
+  for(let j=0; j<viewList.list.length; j++) {
+    let v = viewList.list[j]
+    let viewDetails = []
+
+    // invoke view specific read
+    if (v.type === 1) viewDetails = (await api.dbView.formRead(v.id)).columns;
+    else if (v.type === 2) viewDetails = await api.dbView.galleryRead(v.id);
+    else if (v.type === 3) viewDetails = await api.dbView.gridColumnsList(v.id);
+    viewStore[v.id] = viewDetails;
+  }
+}
+
+// mapping table for quick information access
+// store maps for tableId, columnId, viewColumnId & viewId to their names
+async function generateMapTbl(pId) {
+  const tblList = await api.dbTable.list(pId)
+
+  for(let i=0; i<tblList.list.length; i++) {
+    let tblId = tblList.list[i].id
+    let tbl = await api.dbTable.read(tblId)
+
+    // table ID <> name
+    ncMap[tblId] = tbl.title;
+
+    // column ID <> name
+    tbl.columns.map(x => ncMap[x.id] = x.title )
+
+    // view ID <> name
+    tbl.views.map(x => ncMap[x.id] = x.tn )
+
+    for(let i=0; i<tbl.views.length; i++) {
+      let x = tbl.views[i]
+      let viewColumns = []
+      if (x.type === 1) viewColumns = (await api.dbView.formRead(x.id)).columns;
+      else if (x.type === 2) viewColumns = (await api.dbView.galleryRead(x.id)).columns;
+      else viewColumns = await api.dbView.gridColumnsList(x.id);
+
+      // view column ID <> name
+      viewColumns?.map(a => ncMap[a.id] = ncMap[a.fk_column_id])
+    }
+  }
+}
+
+// main
+//
+async function exportSchema() {
+  api = new Api( ncConfig );
+
+  // fetch project details (id et.al)
+  const x = await api.project.list();
+  const p = x.list.find(a => a.title === ncConfig.projectName);
+
+  await generateMapTbl(p.id)
+
+  // read project
+  const tblList = await api.dbTable.list(p.id)
+  for(let i=0; i<tblList.list.length; i++) {
+    let tblId = tblList.list[i].id
+    await storeViewDetails(tblId)
+
+    let tbl = await api.dbTable.read(tblId)
+
+    // prepare schema
+    let tSchema = {
+      id: tbl.id,
+      title: tbl.title,
+      table_name: tbl?.table_name,
+      columns: [...tbl.columns.map(c => addColumnSpecificData(c))]
+        .filter(a => a.title.includes('_nc_m2m_') === false)
+        .filter(a => !(a?.system===1 && a.uidt===UITypes.LinkToAnotherRecord )),
+      views: [...tbl.views.map(v => addViewDetails(v))]
+    }
+    tblSchema.push(tSchema)
+  }
+}
+
+(async() => {
+  await exportSchema()
+  jsonfile.writeFileSync(`${ncConfig.projectName.replace(/ /g, '_')}.json`, tblSchema, { spaces: 2 })
+})().catch(e => {console.log(e)})
--- a/packages/nocodb/tests/export-import/importSchema.js
+++ b/packages/nocodb/tests/export-import/importSchema.js
@ -0,0 +1,173 @@
+const Api = require('nocodb-sdk').Api;
+const { UITypes } = require('nocodb-sdk');
+const jsonfile = require("jsonfile");
+
+let api = {}
+let ncIn = jsonfile.readFileSync('x.json')
+let ncProject = {}
+let link = []
+let lookup = []
+let rollup = []
+let formula = []
+let ncTables = {}
+
+const ncConfig = {
+  projectName: "x2",
+  baseURL: "http://localhost:8080",
+  headers: {
+    'xc-auth': "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6InVzZXJAbm9jb2RiLmNvbSIsImZpcnN0bmFtZSI6bnVsbCwibGFzdG5hbWUiOm51bGwsImlkIjoidXNfaGJ1aDFmMTNmemc4dTEiLCJyb2xlcyI6InVzZXIsc3VwZXIiLCJpYXQiOjE2NTMwNTU5MzR9.nADVbCbSE0WEbPrpKuq_dlMHrrxieQurYPiOIU2Gf4k"
+  }
+}
+
+async function createBaseTables() {
+  for(let i=0; i<ncIn.length; i++) {
+    let tblSchema = ncIn[i]
+    let reducedColumnSet = tblSchema.columns.filter(a => a.uidt !== UITypes.LinkToAnotherRecord && a.uidt !== UITypes.Lookup && a.uidt !== UITypes.Formula);
+    link.push(...tblSchema.columns.filter(a => a.uidt === UITypes.LinkToAnotherRecord))
+    lookup.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Lookup))
+    rollup.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Rollup))
+    formula.push(...tblSchema.columns.filter(a => a.uidt === UITypes.Formula))
+
+    let tbl = await api.dbTable.create(ncProject.id, {
+      title: tblSchema.title,
+      table_name: tblSchema.title,
+      columns: reducedColumnSet.map(({id,...rest}) => ({...rest}))
+    })
+    ncTables[tbl.title] = tbl;
+    ncTables[tbl.id] = tbl;
+    ncTables[tblSchema.id] = tbl;
+  }
+}
+
+let linksCreated = []
+function isLinkCreated(pId, cId) {
+  let idx = linksCreated.findIndex(a => a.cId === pId && a.pId === cId)
+  if(idx === -1) {
+    linksCreated.push({pId: pId, cId: cId})
+    return false;
+  }
+  return true;
+}
+
+async function createFormula() {
+  for (let i = 0; i < formula.length; i++) {
+    let tbl = await api.dbTableColumn.create(srcTbl.id, {
+      uidt: UITypes.LinkToAnotherRecord,
+      title: link[i].title,
+      parentId: srcTbl.id,
+      childId: dstTbl.id,
+      type: link[i].colOptions.type
+    });
+  }
+}
+
+async function createLinks() {
+  for (let i = 0; i < link.length; i++) {
+    if (((link[i].colOptions.type === 'mm') &&
+        (false === isLinkCreated(link[i].colOptions.fk_parent_column_id, link[i].colOptions.fk_child_column_id)))
+      || (link[i].colOptions.type === 'hm')) {
+      let srcTbl = ncTables[link[i].colOptions.fk_model_id];
+      let dstTbl = ncTables[link[i].colOptions.fk_related_model_id];
+      // create link
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.LinkToAnotherRecord,
+        title: link[i].title,
+        parentId: srcTbl.id,
+        childId: dstTbl.id,
+        type: link[i].colOptions.type
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[link[i].colOptions.fk_model_id] = tbl;
+
+      let v2ColSchema = tbl.columns.find(x => x.title === link[i].title)
+
+      // read related table again after link is created
+      dstTbl = await api.dbTable.read(dstTbl.id);
+      let v2SymmetricColumn = (link[i].colOptions.type === 'mm') ? dstTbl.columns.find(x => x.uidt === UITypes.LinkToAnotherRecord && x?.colOptions.fk_parent_column_id === v2ColSchema.colOptions.fk_child_column_id && x?.colOptions.fk_child_column_id === v2ColSchema.colOptions.fk_parent_column_id) :
+        dstTbl.columns.find(x => x.uidt === UITypes.LinkToAnotherRecord && x?.colOptions.fk_parent_column_id === v2ColSchema.colOptions.fk_parent_column_id && x?.colOptions.fk_child_column_id === v2ColSchema.colOptions.fk_child_column_id)
+      let v1SymmetricColumn = (link[i].colOptions.type === 'mm') ? link.find(x => x.colOptions.fk_parent_column_id === link[i].colOptions.fk_child_column_id && x.colOptions.fk_child_column_id === link[i].colOptions.fk_parent_column_id) :
+        link.find(x => x.colOptions.fk_parent_column_id === link[i].colOptions.fk_parent_column_id && x.colOptions.fk_child_column_id === link[i].colOptions.fk_child_column_id);
+
+      tbl = await api.dbTableColumn.update(v2SymmetricColumn.id, {
+        ...v2SymmetricColumn,
+        title: v1SymmetricColumn.title,
+        column_name: null
+      })
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[v1SymmetricColumn.colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+function get_v2Id(v1ColId) {
+  for(let i=0; i<ncIn.length; i++) {
+    let tblSchema = ncIn[i]
+    let colSchema = {}
+    if(undefined !== (colSchema = tblSchema.columns.find(x => x.id === v1ColId))) {
+      let colName = colSchema.title;
+      let v2Tbl = ncTables[tblSchema.id];
+      return v2Tbl.columns.find(y => y.title === colName)?.id
+    }
+  }
+}
+
+async function createLookup() {
+  for(let i=0; i<lookup.length; i++) {
+    let srcTbl = ncTables[lookup[i].colOptions.fk_model_id];
+    let v2_fk_relation_column_id = get_v2Id(lookup[i].colOptions.fk_relation_column_id)
+    let v2_lookup_column_id = get_v2Id(lookup[i].colOptions.fk_lookup_column_id)
+
+    if(v2_lookup_column_id) {
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.Lookup,
+        title: lookup[i].title,
+        fk_relation_column_id: v2_fk_relation_column_id,
+        fk_lookup_column_id: v2_lookup_column_id
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[lookup[i].colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+async function createRollup() {
+  for(let i=0; i<rollup.length; i++) {
+    let srcTbl = ncTables[rollup[i].colOptions.fk_model_id];
+    let v2_fk_relation_column_id = get_v2Id(rollup[i].colOptions.fk_relation_column_id)
+    let v2_rollup_column_id = get_v2Id(rollup[i].colOptions.fk_rollup_column_id)
+
+    if(v2_rollup_column_id) {
+      let tbl = await api.dbTableColumn.create(srcTbl.id, {
+        uidt: UITypes.Rollup,
+        title: rollup[i].title,
+        fk_relation_column_id: v2_fk_relation_column_id,
+        fk_rollup_column_id: v2_rollup_column_id,
+        rollup_function: rollup[i].colOptions.rollup_function
+      });
+      ncTables[tbl.title] = tbl;
+      ncTables[tbl.id] = tbl;
+      ncTables[rollup[i].colOptions.fk_model_id] = tbl;
+    }
+  }
+}
+
+async function importSchema() {
+  api = new Api(ncConfig);
+
+  const x = await api.project.list();
+  const p = x.list.find(a => a.title === ncConfig.projectName);
+  if (p) await api.project.delete(p.id);
+  ncProject = await api.project.create({ title: ncConfig.projectName })
+  await createBaseTables()
+  await createFormula()
+  await createLinks()
+  await createLookup()
+  await createRollup()
+}
+(async() => {
+  await importSchema()
+  console.log('completed')
+})().catch(e => console.log(e))