RD-10855 RDMBS readers duplicate columns that have the same name

snapi-client/src/main/scala/raw/client/rql2/truffle/Rql2JsonWriter.scala

@@ -140,7 +140,6 @@ class Rql2JsonWriter(os: OutputStream) {
         val index = v.invokeMember("getIndex").asInt()
         val actualValue = v.invokeMember("getValue")
         writeValue(actualValue, tipes(index).asInstanceOf[Rql2TypeWithProperties])
-
       case _ => throw new RuntimeException("unsupported type")
     }
   }

snapi-client/src/test/scala/raw/compiler/rql2/tests/truffle/regressions/RD10767Test.scala

@@ -15,7 +15,9 @@ package raw.compiler.rql2.tests.truffle.regressions
 import raw.client.api.{GetProgramDescriptionSuccess, ProgramEnvironment}
 import raw.compiler.rql2.truffle.TruffleCompilerTestContext
 import raw.compiler.utils.SnapiInterpolator
+import raw.testing.tags.TruffleTests
 
+@TruffleTests
 class RD10767Test extends TruffleCompilerTestContext {
   private val data = tempFile("""
     |[

snapi-client/src/test/scala/raw/compiler/rql2/tests/truffle/regressions/RD10855Test.scala

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2024 RAW Labs S.A.
+ *
+ * Use of this software is governed by the Business Source License
+ * included in the file licenses/BSL.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with
+ * the Business Source License, use of this software will be governed
+ * by the Apache License, Version 2.0, included in the file
+ * licenses/APL.txt.
+ */
+
+package raw.compiler.rql2.tests.truffle.regressions
+
+import raw.compiler.rql2.truffle.TruffleCompilerTestContext
+import raw.creds.jdbc.RDBMSTestCreds
+import raw.testing.tags.TruffleTests
+
+@TruffleTests
+class RD10855Test extends TruffleCompilerTestContext with RDBMSTestCreds {
+
+  // The column y is of type varchar, but we are setting to undefined
+  // when the value is not null it shows the error "skipping column of type ..."
+  test(
+    s"""PostgreSQL.Read(
+      |  "${pgsqlCreds.database}",
+      |  "rdbmstest",
+      |  "tbl1",
+      |   type collection(record(a: int, b: int, c: double, d: double, x: string, y: undefined)),
+      |   host = "${pgsqlCreds.host}",
+      |   username = "${pgsqlCreds.username.get}",
+      |   password = "${pgsqlCreds.password.get}"
+      |)""".stripMargin
+  ) { it =>
+    it should evaluateTo(
+      """[
+        |  {a: 1, b: 1, c: 1.5, d: 1.5, x: "x1", y: "skipping column of type varchar"},
+        |  {a: 2, b: 2, c: 2.2, d: 2.2, x: "x2", y: "skipping column of type varchar"},
+        |  {a: 3, b: null, c: 3.3, d: null, x: "x3", y: null}
+        |]""".stripMargin
+    )
+  }
+
+  // As we are getting items by index, we can rename the columns, as long as the types are correct
+  test(
+    s"""PostgreSQL.Read(
+      |  "${pgsqlCreds.database}",
+      |  "rdbmstest",
+      |  "tbl1",
+      |   type collection(record(e: int, f: int, g: double, h: double, xx: string, yy: string)),
+      |   host = "${pgsqlCreds.host}",
+      |   username = "${pgsqlCreds.username.get}",
+      |   password = "${pgsqlCreds.password.get}"
+      |)""".stripMargin
+  ) { it =>
+    it should evaluateTo(
+      """[
+        |  {e: 1, f: 1, g: 1.5, h: 1.5, xx: "x1", yy: "y1"},
+        |  {e: 2, f: 2, g: 2.2, h: 2.2, xx: "x2", yy: "y2"},
+        |  {e: 3, f: null, g: 3.3, h: null, xx: "x3", yy: null}
+        |]""".stripMargin
+    )
+  }
+
+  // table skippable_types has unsupported types, field f: macaddr, field g: cidr
+  // but are null so no errors will be shown
+  test(
+    s"""PostgreSQL.InferAndRead(
+      |  "${pgsqlCreds.database}",
+      |  "rdbmstest",
+      |  "skippable_types",
+      |   host = "${pgsqlCreds.host}",
+      |   username = "${pgsqlCreds.username.get}",
+      |   password = "${pgsqlCreds.password.get}"
+      |)""".stripMargin
+  ) { it =>
+    it should evaluateTo(
+      """[
+        |  {a: 1, b: "hello", c: 2, d: 3, e: Decimal.From(4.4), f: null, g: null}
+        |]""".stripMargin
+    )
+  }
+
+  val ttt = "\"\"\""
+  // Retrieving columns with repeated names
+  test(s"""PostgreSQL.InferAndQuery(
+    |  "${pgsqlCreds.database}",
+    |  ${ttt}SELECT t1.a, t2.a, t1.b, t2.x as b
+    |    FROM rdbmstest.tbl1 t1, rdbmstest.tbl1 t2
+    |    WHERE t1.a = t2.a $ttt,
+    |   host = "${pgsqlCreds.host}",
+    |   username = "${pgsqlCreds.username.get}",
+    |   password = "${pgsqlCreds.password.get}"
+    |)""".stripMargin) { it =>
+    it should evaluateTo(
+      """[
+        |  {a: 1, a: 1, b: 1, b: "x1"},
+        |  {a: 2, a: 2, b: 2, b: "x2"},
+        |  {a: 3, a: 3, b: null, b: "x3"}
+        |]""".stripMargin
+    )
+  }
+
+  // Interval type is not supported, so its skipped
+  test(s"""PostgreSQL.InferAndQuery(
+    |  "${pgsqlCreds.database}",
+    |  "SELECT * FROM rdbmstest.test_types",
+    |   host = "${pgsqlCreds.host}",
+    |   username = "${pgsqlCreds.username.get}",
+    |   password = "${pgsqlCreds.password.get}"
+    |)""".stripMargin) { it =>
+    it should evaluateTo(
+      """[
+        |  {
+        |    smallint1: 1,
+        |    integer1: 2,
+        |    bigint1: 3,
+        |    decimal1: Decimal.From(4.4),
+        |    real1: 5.5f,
+        |    double1: 6.6,
+        |    money1: 7.7,
+        |    varchar1: "string",
+        |    char1: "string    ",
+        |    text1: "string",
+        |    boolean1: true,
+        |    timestamp1: Timestamp.Build(1975, 6, 23, 1, 2, seconds=3),
+        |    date1: Date.Build(1975, 6, 23),
+        |    time1: Time.Build(1, 2, seconds=3),
+        |    interval1: "skipping column of type interval",
+        |    bytea1: "eyJhIjogMTIzNCwgImIiOiBbImhlbGxvIiwgIndvcmxkIl19",
+        |    bit1: null,
+        |    integer_array1: null,
+        |    smallserial1: 0,
+        |    serial1: 0,
+        |    bigserial1: 0
+        |  }
+        |]""".stripMargin
+    )
+  }
+}

snapi-frontend/src/main/scala/raw/compiler/rql2/Rql2TypeUtils.scala

@@ -98,7 +98,9 @@ trait Rql2TypeUtils {
         )
       case SourceBinaryType(n) => wrapProps(Rql2BinaryType(), n || makeNullable, makeTryable)
       case _: SourceNothingType => wrapProps(Rql2UndefinedType(), makeNullable, makeTryable)
-      case _: SourceNullType => wrapProps(Rql2UndefinedType(), true, makeTryable)
+      // (CTM) Null type will be an error if we find a non null value, so we have to make it triable also.
+      // This is because we use it to skipp unsupported types in databases
+      case _: SourceNullType => wrapProps(Rql2UndefinedType(), true, true)
       case _: SourceAnyType => AnyType()
       case SourceOrType(ors) =>
         val options = ors

snapi-frontend/src/main/scala/raw/compiler/rql2/builtin/MySQLPackage.scala

@@ -12,14 +12,22 @@
 
 package raw.compiler.rql2.builtin
 
+import com.sun.jdi.BooleanValue
 import raw.compiler.base.errors.ErrorCompilerMessage
 import raw.compiler.base.source.{AnythingType, BaseNode, Type}
 import raw.compiler.common.source._
 import raw.compiler.rql2.api._
 import raw.compiler.rql2.ProgramContext
 import raw.compiler.rql2.source._
 import raw.client.api._
-import raw.inferrer.api.{SqlQueryInputFormatDescriptor, SqlTableInputFormatDescriptor}
+import raw.inferrer.api.{
+  SourceAttrType,
+  SourceCollectionType,
+  SourceNullType,
+  SourceRecordType,
+  SqlQueryInputFormatDescriptor,
+  SqlTableInputFormatDescriptor
+}
 
 class MySQLPackage extends PackageExtension {
 
@@ -76,6 +84,18 @@ class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionH
         description =
           """The database user password. Can only to be used together with 'host' and 'username' arguments.""".stripMargin,
         isOptional = true
+      ),
+      ParamDoc(
+        "byIndex",
+        typeDoc = TypeDoc(List("bool")),
+        """Fetch the fields of the database by index instead of by name""".stripMargin,
+        isOptional = true
+      ),
+      ParamDoc(
+        "skipUnsupportedType",
+        typeDoc = TypeDoc(List("bool")),
+        """will remove unsupported types from the output of the query automatically""".stripMargin,
+        isOptional = true
       )
     ),
     examples = List(ExampleDoc("""MySQL.InferAndRead("database", "table")""")),
@@ -98,6 +118,8 @@ class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionH
       case "port" => Right(ValueParam(Rql2IntType()))
       case "username" => Right(ValueParam(Rql2StringType()))
       case "password" => Right(ValueParam(Rql2StringType()))
+      case "byIndex" => Right(ValueParam(Rql2BoolType()))
+      case "skipUnsupportedType" => Right(ValueParam(Rql2BoolType()))
     }
   }
 
@@ -111,7 +133,7 @@ class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionH
     val db = FunAppArg(StringConst(getStringValue(mandatoryArgs(0))), None)
     val table = FunAppArg(StringConst(getStringValue(mandatoryArgs(1))), None)
     val readType = FunAppArg(TypeExp(t), None)
-    val optArgs = optionalArgs.map { case (idn, ValueArg(StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) }
+    val optArgs = optionalArgs.filter(x => x._1 != "skipUnsupportedType").map { case (idn, ValueArg(StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) }
     FunApp(
       Proj(PackageIdnExp("MySQL"), "Read"),
       Vector(db, table, readType) ++ optArgs
@@ -126,9 +148,9 @@ class MySQLInferAndReadEntry extends SugarEntryExtension with SqlTableExtensionH
     for (
       inferrerProperties <- getTableInferrerProperties(mandatoryArgs, optionalArgs, MySqlVendor());
       inputFormatDescriptor <- programContext.infer(inferrerProperties);
-      SqlTableInputFormatDescriptor(_, _, _, _, tipe) = inputFormatDescriptor
+      t <- resolveInferType(inputFormatDescriptor, optionalArgs)
     ) yield {
-      inferTypeToRql2Type(tipe, false, false)
+      t
     }
   }
 }
@@ -183,6 +205,12 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper {
         description =
           """The database user password. Can only to be used together with 'host' and 'username' arguments.""".stripMargin,
         isOptional = true
+      ),
+      ParamDoc(
+        "byIndex",
+        typeDoc = TypeDoc(List("bool")),
+        """Fetch the fields of the database by index instead of by name""".stripMargin,
+        isOptional = true
       )
     ),
     examples =
@@ -207,6 +235,7 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper {
       case "port" => Right(ExpParam(Rql2IntType()))
       case "username" => Right(ExpParam(Rql2StringType()))
       case "password" => Right(ExpParam(Rql2StringType()))
+      case "byIndex" => Right(ExpParam(Rql2BoolType()))
     }
   }
 
@@ -230,7 +259,7 @@ class MySQLReadEntry extends SugarEntryExtension with SqlTableExtensionHelper {
     val db = FunAppArg(mandatoryArgs.head.asInstanceOf[ExpArg].e, None)
     val table = FunAppArg(mandatoryArgs(1).asInstanceOf[ExpArg].e, None)
     val tipe = FunAppArg(TypeExp(mandatoryArgs(2).asInstanceOf[TypeArg].t), None)
-    val optArgs = optionalArgs.map { case (idn, ExpArg(e, _)) => FunAppArg(e, Some(idn)) }
+    val optArgs = optionalArgs.map { case (idn, ValueArg(StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) }
 
     // MySql needs the table name to be quoted with backticks
     def quoted(e: Exp) = BinaryExp(Plus(), BinaryExp(Plus(), StringConst("`"), e), StringConst("`"))
@@ -288,6 +317,18 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension
         description =
           """The database user password. Can only to be used together with 'host' and 'username' arguments.""".stripMargin,
         isOptional = true
+      ),
+      ParamDoc(
+        "byIndex",
+        typeDoc = TypeDoc(List("bool")),
+        """Fetch the fields of the database by index instead of by name""".stripMargin,
+        isOptional = true
+      ),
+      ParamDoc(
+        "skipUnsupportedType",
+        typeDoc = TypeDoc(List("bool")),
+        """will remove unsupported types from the output of the query automatically""".stripMargin,
+        isOptional = true
       )
     ),
     examples = List(ExampleDoc("""MySQL.InferAndQuery("database", "SELECT * FROM table")""")),
@@ -310,6 +351,8 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension
       case "port" => Right(ValueParam(Rql2IntType()))
       case "username" => Right(ValueParam(Rql2StringType()))
       case "password" => Right(ValueParam(Rql2StringType()))
+      case "byIndex" => Right(ValueParam(Rql2BoolType()))
+      case "skipUnsupportedType" => Right(ValueParam(Rql2BoolType()))
     }
   }
 
@@ -321,9 +364,9 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension
     for (
       inferrerProperties <- getQueryInferrerProperties(mandatoryArgs, optionalArgs, MySqlVendor());
       inputFormatDescriptor <- programContext.infer(inferrerProperties);
-      SqlQueryInputFormatDescriptor(_, _, tipe) = inputFormatDescriptor
+      t <- resolveInferType(inputFormatDescriptor, optionalArgs)
     ) yield {
-      inferTypeToRql2Type(tipe, false, false)
+      t
     }
   }
 
@@ -337,7 +380,7 @@ class MySQLInferAndQueryEntry extends SugarEntryExtension with SqlTableExtension
     val db = FunAppArg(StringConst(getStringValue(mandatoryArgs(0))), None)
     val query = FunAppArg(StringConst(getStringValue(mandatoryArgs(1))), None)
     val readType = FunAppArg(TypeExp(t), None)
-    val optArgs = optionalArgs.map { case (idn, ValueArg(StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) }
+    val optArgs = optionalArgs.filter(x => x._1 != "skipUnsupportedType").map { case (idn, ValueArg(StringValue(s), _)) => FunAppArg(StringConst(s), Some(idn)) }
     FunApp(
       Proj(PackageIdnExp("MySQL"), "Query"),
       Vector(db, query, readType) ++ optArgs
@@ -396,6 +439,12 @@ class MySQLQueryEntry extends EntryExtension with SqlTableExtensionHelper {
         description =
           """The database user password. Can only to be used together with 'host' and 'username' arguments.""".stripMargin,
         isOptional = true
+      ),
+      ParamDoc(
+        "byIndex",
+        typeDoc = TypeDoc(List("bool")),
+        """Fetch the fields of the database by index instead of by name""".stripMargin,
+        isOptional = true
       )
     ),
     examples = List(
@@ -413,14 +462,15 @@ class MySQLQueryEntry extends EntryExtension with SqlTableExtensionHelper {
     else Right(ExpParam(Rql2StringType()))
   }
 
-  override def optionalParams: Option[Set[String]] = Some(Set("host", "username", "port", "password"))
+  override def optionalParams: Option[Set[String]] = Some(Set("host", "username", "port", "password", "byIndex"))
 
   override def getOptionalParam(prevMandatoryArgs: Seq[Arg], idn: String): Either[String, Param] = {
     idn match {
       case "host" => Right(ExpParam(Rql2StringType()))
       case "port" => Right(ExpParam(Rql2IntType()))
       case "username" => Right(ExpParam(Rql2StringType()))
       case "password" => Right(ExpParam(Rql2StringType()))
+      case "byIndex" => Right(ExpParam(Rql2BoolType()))
     }
   }
   override def returnTypeErrorList(