HCK-15959: handle string columns collation in RE and FE

forward_engineering/helpers/columnHelper.js

@@ -76,12 +76,10 @@ const getChildBySubtype = (parentType, subtype) => {
 const getPropertyByType = type => {
 	const childTypeDescriptor = getTypeDescriptor(type);
 
-	return Object.assign(
-		{
-			type,
-		},
-		childTypeDescriptor.defaultValues || {},
-	);
+	return {
+		type,
+		...(childTypeDescriptor.defaultValues || {}),
+	};
 };
 
 const getArray = getTypeByProperty => property => {
@@ -105,7 +103,8 @@ const getArray = getTypeByProperty => property => {
 		type = getTypeByProperty(getChildBySubtype('array', property.subtype));
 	}
 
-	return `array<${type}>`;
+	const collation = property.collation ? ` COLLATE ${property.collation}` : '';
+	return `array<${type}${collation}>`;
 };
 
 const getMapKey = property => {
@@ -139,7 +138,9 @@ const getMap = getTypeByProperty => property => {
 		type = getTypeByProperty(getChildBySubtype('map', property.subtype));
 	}
 
-	return `map<${key}, ${type}>`;
+	const collation = property.collation ? ` COLLATE ${property.collation}` : '';
+
+	return `map<${key}, ${type}${collation}>`;
 };
 
 const getText = property => {
@@ -178,7 +179,7 @@ const getJsonType = getTypeByProperty => property => {
 		return 'string';
 	}
 
-	return getTypeByProperty(Object.assign({}, property, { type: property.physicalType }));
+	return getTypeByProperty({ ...property, type: property.physicalType });
 };
 
 const getUnionTypeFromMultiple = getTypeByProperty => property => {
@@ -221,7 +222,7 @@ const getUnionFromAllOf = getTypeByProperty => property => {
 			return types;
 		}
 
-		return Object.assign({}, types, getUnionFromOneOf(getTypeByProperty)(subschema));
+		return { ...types, ...getUnionFromOneOf(getTypeByProperty)(subschema) };
 	}, {});
 };
 
@@ -297,8 +298,17 @@ const getTypeByProperty =
 		}
 	};
 
-const getColumn = (name, type, comment, constraints, isActivated, generatedExpression, maskingFunction) => ({
-	[name]: { type, comment, constraints, isActivated, generatedExpression, maskingFunction },
+const getColumn = ({
+	name,
+	type,
+	comment,
+	constraints,
+	isActivated,
+	generatedExpression,
+	maskingFunction,
+	collation,
+}) => ({
+	[name]: { type, comment, constraints, isActivated, generatedExpression, maskingFunction, collation },
 });
 
 const getGeneratedExpression = (expressionData, defaultValue = '') => {
@@ -353,31 +363,31 @@ const getColumns = (jsonSchema, definitions, dbVersion) => {
 
 		const isPrimaryKey = property.primaryKey && !property.compositePrimaryKey && !property.primaryKeyOptions;
 
-		return Object.assign(
-			{},
-			hash,
-			getColumn(
-				prepareName(name),
-				getTypeByProperty(definitions, dbVersion)(property),
-				getDescription(definitions, property),
-				{
+		return {
+			...hash,
+			...getColumn({
+				name: prepareName(name),
+				type: getTypeByProperty(definitions, dbVersion)(property),
+				comment: getDescription(definitions, property),
+				constraints: {
 					unique: property.unique,
 					...(property.check && getCheckConstraint(property)),
 					...(areNotNullConstraintsAvailable && { notNull: isRequired }),
 					...(arePkFkColumnConstraintsAvailable && { primaryKey: isPrimaryKey }),
 				},
-				property.isActivated,
-				getGeneratedExpression(property.generatedDefaultValue, property.default),
-				property.maskingFunction,
-			),
-		);
+				isActivated: property.isActivated,
+				generatedExpression: getGeneratedExpression(property.generatedDefaultValue, property.default),
+				maskingFunction: property.maskingFunction,
+				collation: property.collation,
+			}),
+		};
 	}, {});
 
 	if (Array.isArray(jsonSchema.oneOf)) {
 		const unions = getUnionFromOneOf(getTypeByProperty(definitions, dbVersion))(jsonSchema);
 
 		columns = Object.keys(unions).reduce(
-			(hash, typeName) => Object.assign({}, hash, getColumn(prepareName(typeName), unions[typeName])),
+			(hash, typeName) => ({ ...hash, ...getColumn({ name: prepareName(typeName), type: unions[typeName] }) }),
 			columns,
 		);
 	}
@@ -386,7 +396,7 @@ const getColumns = (jsonSchema, definitions, dbVersion) => {
 		const unions = getUnionFromAllOf(getTypeByProperty(definitions, dbVersion))(jsonSchema);
 
 		columns = Object.keys(unions).reduce(
-			(hash, typeName) => Object.assign({}, hash, getColumn(prepareName(typeName), unions[typeName])),
+			(hash, typeName) => ({ ...hash, ...getColumn({ name: prepareName(typeName), type: unions[typeName] }) }),
 			columns,
 		);
 	}
@@ -403,14 +413,17 @@ const getColumnStatement = ({
 	isParentActivated,
 	generatedExpression,
 	maskingFunction,
+	collation,
 }) => {
 	const commentStatement = comment ? ` COMMENT '${encodeStringLiteral(comment)}'` : '';
 	const constraintsStatement = constraints ? getColumnConstraintsStatement(constraints) : '';
 	const isColumnActivated = isParentActivated ? isActivated : true;
 	const maskingStatement = maskingFunction ? ` MASK ${maskingFunction}` : '';
+	const isCollationInType = type?.includes(' COLLATE ');
+	const collationStatement = collation && !isCollationInType ? ` COLLATE ${collation}` : '';
 
 	return commentDeactivatedStatements(
-		`${replaceSpaceWithUnderscore(name)} ${type}${generatedExpression}${maskingStatement}${constraintsStatement}${commentStatement}`,
+		`${replaceSpaceWithUnderscore(name)} ${type}${collationStatement}${generatedExpression}${maskingStatement}${constraintsStatement}${commentStatement}`,
 		isColumnActivated,
 	);
 };
@@ -419,7 +432,7 @@ const isCommentedStatement = (statement = '') => statement.startsWith('--');
 
 const getColumnsStatement = (columns, isParentActivated) => {
 	const columnStatements = Object.keys(columns).map(name => {
-		return getColumnStatement(Object.assign({}, columns[name], { name, isParentActivated }));
+		return getColumnStatement({ ...columns[name], name, isParentActivated });
 	});
 
 	const lastColumnStatement = columnStatements[columnStatements.length - 1];

properties_pane/field_level/fieldLevelConfig.json

@@ -137,6 +137,12 @@ making sure that you maintain a proper JSON format.
 				"data": "options",
 				"valueType": "string"
 			},
+			{
+				"propertyName": "Collation",
+				"propertyKeyword": "collation",
+				"propertyTooltip": "collation",
+				"propertyType": "text"
+			},
 			{
 				"propertyName": "JSON Types",
 				"propertyKeyword": "subtype",
@@ -2595,6 +2601,16 @@ making sure that you maintain a proper JSON format.
 					"array<union>"
 				]
 			},
+			{
+				"propertyName": "Collation",
+				"propertyKeyword": "collation",
+				"propertyTooltip": "collation",
+				"propertyType": "text",
+				"dependency": {
+					"key": "subtype",
+					"value": "array<txt>"
+				}
+			},
 			"minItems",
 			"maxItems",
 			"uniqueItems",
@@ -2972,6 +2988,16 @@ making sure that you maintain a proper JSON format.
 					"map<union>"
 				]
 			},
+			{
+				"propertyName": "Value collation",
+				"propertyKeyword": "collation",
+				"propertyTooltip": "collation",
+				"propertyType": "text",
+				"dependency": {
+					"key": "subtype",
+					"value": "map<txt>"
+				}
+			},
 			"minProperties",
 			"maxProperties",
 			{

reverse_engineering/grammars/SqlBase.g4

@@ -937,7 +937,7 @@ dataType
     : complex=ARRAY '<' dataType '>'                            #arrayDataType
     | complex=MAP '<' key=dataType ',' val=dataType '>'         #mapDataType
     | complex=STRUCT ('<' complexColTypeList? '>' | NEQ)        #structDataType
-    | identifier ('(' precision=INTEGER_VALUE (',' scale=INTEGER_VALUE)* ')')?  #primitiveDataType
+    | identifier ('(' precision=INTEGER_VALUE (',' scale=INTEGER_VALUE)* ')')? (COLLATE collation=identifier)?  #primitiveDataType
     ;
 
 qualifiedColTypeWithPositionList
@@ -1148,6 +1148,7 @@ alterColumnAction
     | commentSpec
     | colPosition
     | setOrDrop=(SET | DROP) NOT NULL
+    | COLLATE collation=identifier
     ;
 
 // When `SQL_standard_keyword_behavior=true`, there are 2 kinds of keywords in Spark SQL.

reverse_engineering/helpers/columnsREHelper.js

@@ -117,16 +117,30 @@ const handleType = typeContainer => {
 };
 
 const reverseTableColumn = column => {
+	const collation = getCollation(column);
+
 	return {
 		...handleType(column.colType),
 		name: column.colName,
 		description: column.colComment,
 		default: column.default,
 		...(column.generatedDefaultValue && { generatedDefaultValue: column.generatedDefaultValue }),
 		...(column.primaryKey && { primaryKey: true, primaryKeyOptions: column.primaryKeyOptions }),
+		...(collation && { collation }),
 	};
 };
 
+const getCollation = column => {
+	if (column.colType?.type === 'array') {
+		return column.colType.elements?.collation;
+	}
+	if (column.colType?.type === 'map') {
+		return column.colType.val?.collation;
+	}
+
+	return column.colType?.collation;
+};
+
 /**
  *
  * @param {object[]} properties

reverse_engineering/parser/SQLBase/SqlBaseLexer.js

@@ -1,5 +1,6 @@
 // Generated from grammars/SqlBase.g4 by ANTLR 4.9.2
 // jshint ignore: start
+
 const antlr4 = require('antlr4');
 
 /**
@@ -3694,5 +3695,4 @@ SqlBaseLexer.prototype.BRACKETED_COMMENT_sempred = function (localctx, predIndex
 			throw 'No predicate with index:' + predIndex;
 	}
 };
-
 module.exports = { SqlBaseLexer };