Browse Source

feat: EntityMap handle case-sensitive duplicate column names

Signed-off-by: mertmit <mertmit99@gmail.com>
pull/4156/head
mertmit 2 years ago
parent
commit
6e9ea09a6b
  1. 72
      packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts

72
packages/nocodb/src/lib/meta/api/sync/helpers/EntityMap.ts

@ -8,7 +8,7 @@ class EntityMap {
constructor(...args) { constructor(...args) {
this.initialized = false; this.initialized = false;
this.cols = args; this.cols = args.map((arg) => processKey(arg));
this.db = new Promise((resolve, reject) => { this.db = new Promise((resolve, reject) => {
const db = new sqlite3.Database(':memory:'); const db = new sqlite3.Database(':memory:');
@ -41,14 +41,15 @@ class EntityMap {
throw 'Please initialize first!'; throw 'Please initialize first!';
} }
const colStatement = Object.keys(row).map((key) => `'${key.replace(/'/gi, "''")}'`).join(', '); const cols = Object.keys(row).map((key) => processKey(key));
const questionMarks = Object.keys(row).map(() => '?').join(', '); const colStatement = cols.map((key) => `'${key}'`).join(', ');
const questionMarks = cols.map(() => '?').join(', ');
const promises = []; const promises = [];
for (const col of Object.keys(row).filter((col) => !this.cols.includes(col))) { for (const col of cols.filter((col) => !this.cols.includes(col))) {
promises.push(new Promise((resolve, reject) => { promises.push(new Promise((resolve, reject) => {
this.db.run(`ALTER TABLE mapping ADD '${col.replace(/'/gi, "''")}' TEXT;`, (err) => { this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => {
if (err) { if (err) {
console.log(err); console.log(err);
reject(err); reject(err);
@ -84,21 +85,15 @@ class EntityMap {
throw 'Please initialize first!'; throw 'Please initialize first!';
} }
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
col = processKey(col);
res = res.map((r) => processKey(r));
this.db.get(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping WHERE ${col} = ?`, [val], (err, rs) => { this.db.get(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping WHERE ${col} = ?`, [val], (err, rs) => {
if (err) { if (err) {
console.log(err); console.log(err);
reject(err); reject(err);
} }
if (rs) { if (rs) {
for (const key of Object.keys(rs)) { rs = processResponseRow(rs);
if (rs[key] && rs[key].startsWith('JSON::')) {
try {
rs[key] = JSON.parse(rs[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
}
}
} }
resolve(rs) resolve(rs)
}); });
@ -124,6 +119,7 @@ class EntityMap {
if (!this.initialized) { if (!this.initialized) {
throw 'Please initialize first!'; throw 'Please initialize first!';
} }
res = res.map((r) => processKey(r));
return new DBStream(this.db, `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`); return new DBStream(this.db, `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping`);
} }
@ -132,21 +128,14 @@ class EntityMap {
throw 'Please initialize first!'; throw 'Please initialize first!';
} }
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
res = res.map((r) => processKey(r));
this.db.all(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping LIMIT ${limit} OFFSET ${offset}`, (err, rs) => { this.db.all(`SELECT ${res.length ? res.join(', ') : '*'} FROM mapping LIMIT ${limit} OFFSET ${offset}`, (err, rs) => {
if (err) { if (err) {
console.log(err); console.log(err);
reject(err); reject(err);
} }
for (const row of rs) { for (let row of rs) {
for (const key of Object.keys(row)) { row = processResponseRow(row);
if (row[key] && row[key].startsWith('JSON::')) {
try {
row[key] = JSON.parse(row[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
}
}
} }
resolve(rs) resolve(rs)
}); });
@ -174,15 +163,7 @@ class DBStream extends Readable {
stream.emit('error', err); stream.emit('error', err);
} else { } else {
if (result) { if (result) {
for (const key of Object.keys(result)) { result = processResponseRow(result);
if (result[key] && result[key].startsWith('JSON::')) {
try {
result[key] = JSON.parse(result[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
}
}
} }
stream.push(result || null) stream.push(result || null)
} }
@ -190,4 +171,29 @@ class DBStream extends Readable {
} }
} }
function processResponseRow(res: any) {
for (const key of Object.keys(res)) {
if (res[key] && res[key].startsWith('JSON::')) {
try {
res[key] = JSON.parse(res[key].replace('JSON::', ''));
} catch (e) {
console.log(e);
}
}
if (revertKey(key) !== key) {
res[revertKey(key)] = res[key];
delete res[key];
}
}
return res;
}
function processKey(key) {
return key.replace(/'/gi, "''").replace(/[A-Z]/g, (match) => `_${match}`);
}
function revertKey(key) {
return key.replace(/''/gi, "'").replace(/_[A-Z]/g, (match) => match[1]);
}
export default EntityMap; export default EntityMap;

Loading…
Cancel
Save