Skip to content

Commit 38ce4e7

Browse files
committed
[SPARK-23938][SQL] Addressing review comments
1 parent 6aeaaa8 commit 38ce4e7

1 file changed

Lines changed: 53 additions & 60 deletions

File tree

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala‎

Lines changed: 53 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
524524
}
525525

526526
@transient private lazy val getKeysWithValueIndexes:
527-
(ArrayData, ArrayData) => Seq[(Any, Array[Option[Int]])] = {
527+
(ArrayData, ArrayData) => mutable.Iterable[(Any, (Option[Int], Option[Int]))] = {
528528
if (keyTypeSupportsEquals) {
529529
getKeysWithIndexesFast
530530
} else {
@@ -541,83 +541,76 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
541541
}
542542

543543
private def getKeysWithIndexesFast(keys1: ArrayData, keys2: ArrayData) = {
544-
val arrayBuffer = new mutable.ArrayBuffer[(Any, Array[Option[Int]])]
545-
val hashMap = new mutable.OpenHashMap[Any, Array[Option[Int]]]
546-
val keys = Array(keys1, keys2)
547-
var z = 0
548-
while(z < 2) {
549-
var i = 0
550-
val array = keys(z)
551-
while (i < array.numElements()) {
552-
val key = array.get(i, keyType)
553-
hashMap.get(key) match {
554-
case Some(indexes) =>
555-
if (indexes(z).isEmpty) indexes(z) = Some(i)
556-
case None =>
557-
assertSizeOfArrayBuffer(arrayBuffer.size)
558-
val indexes = Array[Option[Int]](None, None)
559-
indexes(z) = Some(i)
560-
hashMap.put(key, indexes)
561-
arrayBuffer += Tuple2(key, indexes)
562-
}
563-
i += 1
544+
val hashMap = new mutable.OpenHashMap[Any, (Option[Int], Option[Int])]
545+
var i = 0
546+
while (i < keys1.numElements) {
547+
val key = keys1.get(i, keyType)
548+
if(!hashMap.contains(key)) hashMap.put(key, (Some(i), None))
549+
i += 1
550+
}
551+
i = 0
552+
while (i < keys2.numElements) {
553+
val key = keys2.get(i, keyType)
554+
hashMap.get(key) match {
555+
case Some((index1, index2)) =>
556+
if (index2.isEmpty) hashMap.update(key, (index1, Some(i)))
557+
case None =>
558+
hashMap.put(key, (None, Some(i)))
564559
}
565-
z += 1
560+
i += 1
566561
}
567-
arrayBuffer
562+
hashMap
568563
}
569564

570565
private def getKeysWithIndexesBruteForce(keys1: ArrayData, keys2: ArrayData) = {
571-
val arrayBuffer = new mutable.ArrayBuffer[(Any, Array[Option[Int]])]
572-
val keys = Array(keys1, keys2)
573-
var z = 0
574-
while(z < 2) {
575-
var i = 0
576-
val array = keys(z)
577-
while (i < array.numElements()) {
578-
val key = array.get(i, keyType)
579-
var found = false
580-
var j = 0
581-
while (!found && j < arrayBuffer.size) {
582-
val (bufferKey, indexes) = arrayBuffer(j)
583-
if (ordering.equiv(bufferKey, key)) {
584-
found = true
585-
if(indexes(z).isEmpty) indexes(z) = Some(i)
586-
}
587-
j += 1
588-
}
589-
if (!found) {
590-
assertSizeOfArrayBuffer(arrayBuffer.size)
591-
val indexes = Array[Option[Int]](None, None)
592-
indexes(z) = Some(i)
593-
arrayBuffer += Tuple2(key, indexes)
566+
val arrayBuffer = new mutable.ArrayBuffer[(Any, (Option[Int], Option[Int]))]
567+
var i = 0
568+
while (i < keys1.numElements) {
569+
val key = keys1.get(i, keyType)
570+
var found = false
571+
var j = 0
572+
while (!found && j < arrayBuffer.size) {
573+
if (ordering.equiv(arrayBuffer(j)._1, key)) found = true
574+
j += 1
575+
}
576+
if (!found) arrayBuffer += Tuple2(key, (Some(i), None))
577+
i += 1
578+
}
579+
i = 0
580+
while (i < keys2.numElements) {
581+
val key = keys2.get(i, keyType)
582+
var found = false
583+
var j = 0
584+
while (!found && j < arrayBuffer.size) {
585+
val (bufferKey, (index1, index2)) = arrayBuffer(j)
586+
if (ordering.equiv(bufferKey, key)) {
587+
found = true
588+
if(index2.isEmpty) arrayBuffer(j) = Tuple2(key, (index1, Some(i)))
594589
}
595-
i += 1
590+
j += 1
596591
}
597-
z += 1
592+
if (!found) {
593+
assertSizeOfArrayBuffer(arrayBuffer.size)
594+
arrayBuffer += Tuple2(key, (None, Some(i)))
595+
}
596+
i += 1
598597
}
599598
arrayBuffer
600599
}
601600

602-
private def getValue(valueData: ArrayData, eType: DataType, index: Option[Int]) = index match {
603-
case Some(i) => valueData.get(i, eType)
604-
case None => null
605-
}
606-
607601
private def nullSafeEval(inputRow: InternalRow, value1: Any, value2: Any): Any = {
608602
val mapData1 = value1.asInstanceOf[MapData]
609603
val mapData2 = value2.asInstanceOf[MapData]
610604
val keysWithIndexes = getKeysWithValueIndexes(mapData1.keyArray(), mapData2.keyArray())
611-
val length = keysWithIndexes.length
612-
val keys = new GenericArrayData(new Array[Any](length))
613-
val values = new GenericArrayData(new Array[Any](length))
605+
val size = keysWithIndexes.size
606+
val keys = new GenericArrayData(new Array[Any](size))
607+
val values = new GenericArrayData(new Array[Any](size))
614608
val valueData1 = mapData1.valueArray()
615609
val valueData2 = mapData2.valueArray()
616610
var i = 0
617-
while(i < length) {
618-
val (key, indexes) = keysWithIndexes(i)
619-
val v1 = getValue(valueData1, leftValueType, indexes(0))
620-
val v2 = getValue(valueData2, rightValueType, indexes(1))
611+
keysWithIndexes.foreach { case (key, (index1, index2)) =>
612+
val v1 = index1.map(valueData1.get(_, leftValueType)).getOrElse(null)
613+
val v2 = index2.map(valueData2.get(_, rightValueType)).getOrElse(null)
621614
keyVar.value.set(key)
622615
value1Var.value.set(v1)
623616
value2Var.value.set(v2)

0 commit comments

Comments
 (0)