[Paused] Enable ODBC build in macOS Intel CI

.github/workflows/cpp.yml

@@ -229,7 +229,34 @@ jobs:
           # pkg-config formula is removed from GitHub Actions runner.
           brew uninstall pkg-config || :
           brew uninstall [email protected] || :
+
+          # -AL- remove build-in SQL headers
+          ls -l /usr/local/include/sqlext.h
+          sudo rm -f /usr/local/include/sql.h \
+          /usr/local/include/sqlext.h \
+          /usr/local/include/sqltypes.h \
+          /usr/local/include/sqlucode.h || true
+
           brew bundle --file=cpp/Brewfile
+
+          # -AL- in case sqlext.h still exists
+          ls -l /usr/local/include/sqlext.h || true
+          # -AL- check real path of libodbc
+          # brew list libiodbc
+      - name: Enable ODBC build on 15-intel only
+        if: ${{ matrix.macos-version == '15-intel' }}
+        run: |
+          echo "ARROW_FLIGHT_SQL=ON" >> $GITHUB_ENV
+          echo "ARROW_FLIGHT_SQL_ODBC=ON" >> $GITHUB_ENV
+          echo "ODBC_INCLUDE_DIR=$(brew --prefix unixodbc)/include" >> $GITHUB_ENV
+          echo "ODBC_LIBRARY=$(brew --prefix unixodbc)/lib/libodbc.2.dylib" >> $GITHUB_ENV
+
+          # -AL- old code to use libiodbc
+          # echo "ODBC_INCLUDE_DIR=$(brew --prefix libiodbc)/include" >> $GITHUB_ENV
+          # echo "ODBC_LIBRARY=$(brew --prefix libiodbc)/lib/libiodbc.dylib" >> $GITHUB_ENV
+          # -AL- check real path of libodbc
+          realpath /usr/local/lib/libodbc.dylib
+
       - name: Install MinIO
         run: |
           $(brew --prefix bash)/bin/bash \
@@ -271,13 +298,14 @@ jobs:
             export BUILD_WARNING_LEVEL=PRODUCTION
           fi
           ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
-      - name: Test
-        shell: bash
-        run: |
-          sudo sysctl -w kern.coredump=1
-          sudo sysctl -w kern.corefile=/tmp/core.%N.%P
-          ulimit -c unlimited  # must enable within the same shell
-          ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
+      # -AL- temporarily disable test phase. Enable after script registration is done.
+      # - name: Test
+      #   shell: bash
+      #   run: |
+      #     sudo sysctl -w kern.coredump=1
+      #     sudo sysctl -w kern.corefile=/tmp/core.%N.%P
+      #     ulimit -c unlimited  # must enable within the same shell
+      #     ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
 
   windows:
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
@@ -390,6 +418,10 @@ jobs:
           PIPX_BASE_PYTHON: ${{ steps.python-install.outputs.python-path }}
         run: |
           ci/scripts/install_gcs_testbench.sh default
+      - name: Register Flight SQL ODBC Driver
+        shell: cmd
+        run: |
+          call "cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd" ${{ github.workspace }}\build\cpp\%ARROW_BUILD_TYPE%\libarrow_flight_sql_odbc.dll
       - name: Test
         shell: msys2 {0}
         run: |

ci/scripts/cpp_build.sh

@@ -272,6 +272,8 @@ else
     -DGTest_SOURCE=${GTest_SOURCE:-} \
     -Dlz4_SOURCE=${lz4_SOURCE:-} \
     -Dopentelemetry-cpp_SOURCE=${opentelemetry_cpp_SOURCE:-} \
+    -DODBC_INCLUDE_DIR=${ODBC_INCLUDE_DIR:-} \
+    -DODBC_LIBRARY=${ODBC_LIBRARY:-} \
     -DORC_SOURCE=${ORC_SOURCE:-} \
     -DPARQUET_BUILD_EXAMPLES=${PARQUET_BUILD_EXAMPLES:-OFF} \
     -DPARQUET_BUILD_EXECUTABLES=${PARQUET_BUILD_EXECUTABLES:-OFF} \

cpp/Brewfile

@@ -28,6 +28,8 @@ brew "git"
 brew "glog"
 brew "googletest"
 brew "grpc"
+# -AL- remove libiodbc to try unixodbc
+# brew "libiodbc"
 brew "llvm"
 brew "lz4"
 brew "mimalloc"
@@ -41,6 +43,7 @@ brew "rapidjson"
 brew "re2"
 brew "snappy"
 brew "thrift"
+brew "unixodbc"
 brew "utf8proc"
 brew "wget"
 brew "xsimd"

cpp/cmake_modules/DefineOptions.cmake

@@ -107,8 +107,8 @@ macro(tsort_bool_option_dependencies)
 endmacro()
 
 macro(resolve_option_dependencies)
-  # Arrow Flight SQL ODBC is available only for Windows for now.
-  if(NOT WIN32)
+  # Arrow Flight SQL ODBC is available only for Windows and macOS for now.
+  if(NOT WIN32 AND NOT APPLE)
     set(ARROW_FLIGHT_SQL_ODBC OFF)
   endif()
   if(MSVC_TOOLCHAIN)

cpp/src/arrow/flight/sql/odbc/README

@@ -30,9 +30,9 @@ After the build succeeds, the ODBC DLL will be located in
     i. `cd to repo.`
     ii. `cd <path\to\repo>`
     iii. Run script to register your ODBC DLL as Apache Arrow Flight SQL ODBC Driver
-    `.\cpp\src\arrow\flight\sql\odbc\install\install_amd64.cmd <path\to\repo>\cpp\build\< release | debug >\< Release | Debug>\arrow_flight_sql_odbc.dll`
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd <path\to\repo>\cpp\build\< release | debug >\< Release | Debug>\arrow_flight_sql_odbc.dll`
     Example command for reference:
-    `.\cpp\src\arrow\flight\sql\odbc\install\install_amd64.cmd C:\path\to\arrow\cpp\build\release\Release\arrow_flight_sql_odbc.dll`
+    `.\cpp\src\arrow\flight\sql\odbc\tests\install_odbc.cmd C:\path\to\arrow\cpp\build\release\Release\arrow_flight_sql_odbc.dll`
 
 If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
 should show as an available ODBC driver in the x64 ODBC Driver Manager.
@@ -44,6 +44,27 @@ should show as an available ODBC driver in the x64 ODBC Driver Manager.
 
 If the generation is successful, you will find `Apache Arrow Flight SQL ODBC-<version>-win64.msi` generated under the `build` folder.
 
+## Steps to Register the 64-bit Apache Arrow ODBC driver on macOS
+
+After the build succeeds, the ODBC DLL will be located in 
+`build\debug` for a debug build and `build\release` for a release build.
+
+1. Open terminal shell.
+
+2. Register your ODBC DYLIB:
+    Need to replace <path\to\repo> with actual path to repository in the commands.
+
+    i. `cd to repo.`
+    ii. `cd <path\to\repo>`
+    iii. Give script permission to execute
+    `chmod +x cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh`
+    iv. Run script with `sudo` to register your ODBC DYLIB as Apache Arrow Flight SQL ODBC Driver
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh <path\to\repo>/cpp/build/< release | debug >/libarrow_flight_sql_odbc.dylib`
+    Example command for reference:
+    `sudo cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh /path/to/arrow/cpp/build/release/libarrow_flight_sql_odbc.dylib`
+
+If the registration is successful, then Apache Arrow Flight SQL ODBC Driver 
+should be shown at `~/Library/ODBC/odbcinst.ini`
 
 ## Steps to Enable Logging
 Arrow Flight SQL ODBC driver uses Arrow's internal logging framework. By default, the log messages are printed to the terminal.
@@ -60,3 +81,8 @@ The characters are case-insensitive.
 The Windows ODBC driver currently does not support writing log files. `ARROW_USE_GLOG` is required to write log files, and `ARROW_USE_GLOG` is disabled on Windows platform since plasma using `glog` is not fully tested on windows.
 
 Note: GH-47670 running more than 1 tests with logging enabled is not fully supported. 
+
+## macOS Intel Build Trouble-shooting
+1. Cmake cannot find `libiodbc` from brew library
+
+  If there are multiple odbc libraries installed on your machine, cmake might not find the `libiodbc` library. Pass cmake flags `-DODBC_INCLUDE_DIR=/usr/local/Cellar/libiodbc/3.52.16/include -DODBC_LIBRARY=/usr/local/Cellar/libiodbc/3.52.16/lib/libiodbc.dylib` in the cmake configuration phase.

cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc.sh

@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Used by macOS ODBC installer script and macOS ODBC testing
+
+ODBC_64BIT="$1"
+
+if [[ -z "$ODBC_64BIT" ]]; then
+  echo "error: 64-bit driver is not specified. Call format: install_odbc abs_path_to_64_bit_driver"
+  exit 1
+fi
+
+if [ ! -f $ODBC_64BIT ]; then
+    echo "64-bit driver can not be found: $ODBC_64BIT"
+	  echo "Call format: install_odbc abs_path_to_64_bit_driver"
+    exit 1
+fi
+
+USER_ODBCINST_FILE="$HOME/Library/ODBC/odbcinst.ini"
+DRIVER_NAME="Apache Arrow Flight SQL ODBC Driver"
+DSN_NAME="Apache Arrow Flight SQL ODBC DSN"
+
+touch "$USER_ODBCINST_FILE"
+
+# Admin privilege is needed to add ODBC driver registration
+if [ $EUID -ne 0 ]; then 
+    echo "Please run this script with sudo"
+    exit 1
+fi
+
+if grep -q "^\[$DRIVER_NAME\]" "$USER_ODBCINST_FILE"; then
+  echo "Driver [$DRIVER_NAME] already exists in odbcinst.ini"
+else
+  echo "Adding [$DRIVER_NAME] to odbcinst.ini..."
+  echo "
+[$DRIVER_NAME]
+Description=An ODBC Driver for Apache Arrow Flight SQL
+Driver=$ODBC_64BIT
+" >> "$USER_ODBCINST_FILE"
+fi
+
+# Check if [ODBC Drivers] section exists
+if grep -q '^\[ODBC Drivers\]' "$USER_ODBCINST_FILE"; then
+  # Section exists: check if driver entry exists
+  if ! grep -q "^${DRIVER_NAME}=" "$USER_ODBCINST_FILE"; then
+    # Driver entry does not exist, add under [ODBC Drivers]
+    sed -i '' "/^\[ODBC Drivers\]/a\\
+${DRIVER_NAME}=Installed
+" "$USER_ODBCINST_FILE"
+  fi
+else
+  # Section doesn't exist, append both section and driver entry at end
+  {
+    echo ""
+    echo "[ODBC Drivers]"
+    echo "${DRIVER_NAME}=Installed"
+  } >> "$USER_ODBCINST_FILE"
+fi

cpp/src/arrow/flight/sql/odbc/install/mac/install_odbc_ini.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# GH-47876 TODO: create macOS ODBC Installer.
+# Script for installing macOS ODBC driver, to be used for macOS installer.
+
+source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+odbc_install_script="${source_dir}/install_odbc.sh"
+
+chmod +x "$odbc_install_script"
+. "$odbc_install_script" /Library/Apache/ArrowFlightSQLODBC/lib/libarrow_flight_sql_odbc.dylib
+
+USER_ODBC_FILE="$HOME/Library/ODBC/odbc.ini"
+DRIVER_NAME="Apache Arrow Flight SQL ODBC Driver"
+DSN_NAME="Apache Arrow Flight SQL ODBC DSN"
+
+touch "$USER_ODBC_FILE"
+
+if [ $EUID -ne 0 ]; then 
+    echo "Please run this script with sudo"
+    exit 1
+fi
+
+if grep -q "^\[$DSN_NAME\]" "$USER_ODBC_FILE"; then
+  echo "DSN [$DSN_NAME] already exists in $USER_ODBC_FILE"
+else
+  echo "Adding [$DSN_NAME] to $USER_ODBC_FILE..."
+  cat >> "$USER_ODBC_FILE" <<EOF
+
+[$DSN_NAME]
+Description = An ODBC Driver DSN for Apache Arrow Flight SQL
+Driver      = Apache Arrow Flight SQL ODBC Driver
+Host        =
+Port        =
+UID         =
+PWD         =
+EOF
+fi
+
+# Check if [ODBC Data Sources] section exists
+if grep -q '^\[ODBC Data Sources\]' "$USER_ODBC_FILE"; then
+  # Section exists: check if DSN entry exists
+  if ! grep -q "^${DSN_NAME}=" "$USER_ODBC_FILE"; then
+    # Add DSN entry under [ODBC Data Sources] section
+
+    # Use awk to insert the line immediately after [ODBC Data Sources]
+    awk -v dsn="$DSN_NAME" -v driver="$DRIVER_NAME" '
+      $0 ~ /^\[ODBC Data Sources\]/ && !inserted {
+        print
+        print dsn "=" driver
+        inserted=1
+        next
+      }
+      { print }
+    ' "$USER_ODBC_FILE" > "${USER_ODBC_FILE}.tmp" && mv "${USER_ODBC_FILE}.tmp" "$USER_ODBC_FILE"
+  fi
+else
+  # Section doesn't exist, append section and DSN entry at end
+  {
+    echo ""
+    echo "[ODBC Data Sources]"
+    echo "${DSN_NAME}=${DRIVER_NAME}"
+  } >> "$USER_ODBC_FILE"
+fi
+

cpp/src/arrow/flight/sql/odbc/odbc_api.cc

@@ -853,7 +853,7 @@ SQLRETURN SQLDriverConnect(SQLHDBC conn, SQLHWND window_handle,
     }
 #else
     // Attempt connection without loading DSN window on macOS/Linux
-    connection->Connect(dsn, properties, missing_properties);
+    connection->Connect(dsn_value, properties, missing_properties);
 #endif
     // Copy connection string to out_connection_string after connection attempt
     return ODBC::GetStringAttribute(true, connection_string, false, out_connection_string,
@@ -882,7 +882,7 @@ SQLRETURN SQLConnect(SQLHDBC conn, SQLWCHAR* dsn_name, SQLSMALLINT dsn_name_len,
     ODBCConnection* connection = reinterpret_cast<ODBCConnection*>(conn);
     std::string dsn = SqlWcharToString(dsn_name, dsn_name_len);
 
-    Configuration config;
+    config::Configuration config;
     config.LoadDsn(dsn);
 
     if (user_name) {

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/binary_array_accessor.h

@@ -18,8 +18,8 @@
 #pragma once
 
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {
 

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/boolean_array_accessor.h

@@ -19,8 +19,8 @@
 
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
 #include "arrow/flight/sql/odbc/odbc_impl/diagnostics.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {
 

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/date_array_accessor.h

@@ -18,8 +18,8 @@
 #pragma once
 
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {
 

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/decimal_array_accessor.h

@@ -19,9 +19,9 @@
 
 #include <locale>
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
 #include "arrow/flight/sql/odbc/odbc_impl/util.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {
 

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/string_array_accessor.h

@@ -20,9 +20,9 @@
 #include <locale>
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
 #include "arrow/flight/sql/odbc/odbc_impl/encoding.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
 #include "arrow/flight/sql/odbc/odbc_impl/util.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {
 

cpp/src/arrow/flight/sql/odbc/odbc_impl/accessors/time_array_accessor.h

@@ -18,8 +18,8 @@
 #pragma once
 
 #include "arrow/flight/sql/odbc/odbc_impl/accessors/types.h"
+#include "arrow/flight/sql/odbc/odbc_impl/type_fwd_fixup.h"
 #include "arrow/flight/sql/odbc/odbc_impl/types.h"
-#include "arrow/type_fwd.h"
 
 namespace arrow::flight::sql::odbc {