diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index dc661a73..d42e0a16 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -271,29 +271,29 @@ def _get_numeric_data(self, param: decimal.Decimal) -> Any: num_digits = len(digits_tuple) exponent = decimal_as_tuple.exponent - # Handle special values (NaN, Infinity, etc.) + # NaN / sNaN / Infinity report a string exponent ('n', 'N', 'F') instead of an + # int. There is no SQL NUMERIC encoding for them, so refuse here rather than + # falling through to precision=38 and letting the digit packing below emit a + # silent zero. The native detection path raises ValueError for the same input. if isinstance(exponent, str): - # For special values like 'n' (NaN), 'N' (sNaN), 'F' (Infinity) - # Return default precision and scale - precision = 38 # SQL Server default max precision + raise ValueError("Cannot bind non-finite Decimal (NaN/Infinity) as SQL NUMERIC") + + # Calculate the SQL precision & scale + # precision = no. of significant digits + # scale = no. digits after decimal point + if exponent >= 0: + # digits=314, exp=2 ---> '31400' --> precision=5, scale=0 + precision = num_digits + exponent scale = 0 + elif (-1 * exponent) <= num_digits: + # digits=3140, exp=-3 ---> '3.140' --> precision=4, scale=3 + precision = num_digits + scale = exponent * -1 else: - # Calculate the SQL precision & scale - # precision = no. of significant digits - # scale = no. digits after decimal point - if exponent >= 0: - # digits=314, exp=2 ---> '31400' --> precision=5, scale=0 - precision = num_digits + exponent - scale = 0 - elif (-1 * exponent) <= num_digits: - # digits=3140, exp=-3 ---> '3.140' --> precision=4, scale=3 - precision = num_digits - scale = exponent * -1 - else: - # digits=3140, exp=-5 ---> '0.03140' --> precision=5, scale=5 - # TODO: double check the precision calculation here with SQL documentation - precision = exponent * -1 - scale = exponent * -1 + # digits=3140, exp=-5 ---> '0.03140' --> precision=5, scale=5 + # TODO: double check the precision calculation here with SQL documentation + precision = exponent * -1 + scale = exponent * -1 if precision > 38: raise ValueError( @@ -511,27 +511,29 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg num_digits = len(digits_tuple) exponent = decimal_as_tuple.exponent - # Handle special values (NaN, Infinity, etc.) + # NaN / sNaN / Infinity report a string exponent ('n', 'N', 'F'). Reject them + # before the MONEY range comparison below, which would otherwise raise + # decimal.InvalidOperation for NaN, and before _get_numeric_data, which used to + # fail with TypeError for Infinity. The native detection path raises ValueError + # for the same input, so both paths now agree on type and message. if isinstance(exponent, str): logger.debug( - "_map_sql_type: DECIMAL special value - index=%d, exponent=%s", i, exponent + "_map_sql_type: DECIMAL non-finite value - index=%d, exponent=%s", i, exponent ) - # For special values like 'n' (NaN), 'N' (sNaN), 'F' (Infinity) - # Return default precision and scale - precision = 38 # SQL Server default max precision + raise ValueError("Cannot bind non-finite Decimal (NaN/Infinity) as SQL NUMERIC") + + # Calculate the SQL precision (same logic as _get_numeric_data) + if exponent >= 0: + precision = num_digits + exponent + elif (-1 * exponent) <= num_digits: + precision = num_digits else: - # Calculate the SQL precision (same logic as _get_numeric_data) - if exponent >= 0: - precision = num_digits + exponent - elif (-1 * exponent) <= num_digits: - precision = num_digits - else: - precision = exponent * -1 - logger.debug( - "_map_sql_type: DECIMAL precision calculated - index=%d, precision=%d", - i, - precision, - ) + precision = exponent * -1 + logger.debug( + "_map_sql_type: DECIMAL precision calculated - index=%d, precision=%d", + i, + precision, + ) if precision > 38: logger.debug( @@ -1001,6 +1003,14 @@ def _create_parameter_types_list( # pylint: disable=too-many-arguments,too-many ) -> Tuple[int, int, int, int, bool]: """ Maps parameter types for the given parameter. + + Python-side type detection. The standard execute() path no longer calls this — + DetectParamTypes does the same job in C++ without crossing the pybind11 + boundary per parameter. Two callers remain: the legacy execute() branch used + when setinputsizes overrides are active, and executemany(). The former goes + away once setinputsizes is handled natively; the latter needs its own + native columnwise detection before this can be deleted outright. + Args: parameter: parameter to bind. Returns: @@ -1522,11 +1532,6 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Getting encoding setting encoding_settings = self._get_encoding_settings() - # Apply timeout if set (non-zero) - logger.debug("execute: Creating parameter type list") - param_info = ddbc_bindings.ParamInfo - parameters_type = [] - # Validate that inputsizes matches parameter count if both are present if parameters and self._inputsizes: if len(self._inputsizes) != len(parameters): @@ -1538,11 +1543,6 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state Warning, ) - if parameters: - for i, param in enumerate(parameters): - paraminfo = self._create_parameter_types_list(param, param_info, parameters, i) - parameters_type.append(paraminfo) - # Prepare caching: skip SQLPrepare when re-executing the same SQL # with parameters. The HSTMT is reused via _soft_reset_cursor, so the # server-side plan from the previous SQLPrepare is still valid. @@ -1551,30 +1551,64 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self.is_stmt_prepared = [False] effective_use_prepare = use_prepare and not same_sql - if logger.isEnabledFor(logging.DEBUG): - for i, param in enumerate(parameters): - logger.debug( - """Parameter number: %s, Parameter: %s, - Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""", - i + 1, - param, - str(type(param)), - parameters_type[i].paramSQLType, - parameters_type[i].paramCType, - parameters_type[i].columnSize, - parameters_type[i].decimalDigits, - parameters_type[i].inputOutputType, - ) - - ret = ddbc_bindings.DDBCSQLExecute( - self.hstmt, - operation, - parameters, - parameters_type, - self.is_stmt_prepared, - effective_use_prepare, - encoding_settings, + # Standard path: when no inputsizes override, type detection + bind + execute + # all happen in C++ via DDBCSQLExecute. ParamInfo never crosses the pybind11 + # boundary. This is the path ~99% of calls take. + use_standard_execute = parameters and not ( + self._inputsizes and any(s is not None for s in self._inputsizes) ) + + if use_standard_execute: + ret = ddbc_bindings.DDBCSQLExecute( + self.hstmt, + operation, + parameters, + self.is_stmt_prepared, + effective_use_prepare, + encoding_settings, + ) + else: + # LEGACY PATH — slated for removal in a future optimization round. + # + # Kept only for setinputsizes() callers, where the user's explicit type + # overrides have to be honoured instead of C++ detecting types itself. + # Type detection happens in Python here, so every parameter round-trips + # through pybind11 as a ParamInfo object, which is what makes it slow. + # Once setinputsizes overrides are handled natively, this branch and + # DDBCSQLExecuteLegacy both go away. + parameters_type = [] + if parameters: + param_info = ddbc_bindings.ParamInfo + for i, param in enumerate(parameters): + paraminfo = self._create_parameter_types_list(param, param_info, parameters, i) + parameters_type.append(paraminfo) + + if logger.isEnabledFor(logging.DEBUG): + for i, param in enumerate(parameters): + logger.debug( + """Parameter number: %s, Parameter: %s, + Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""", + i + 1, + param, + str(type(param)), + parameters_type[i].paramSQLType, + parameters_type[i].paramCType, + parameters_type[i].columnSize, + parameters_type[i].decimalDigits, + parameters_type[i].inputOutputType, + ) + + # Legacy binding: accepts the pre-built ParamInfo list from Python. + # Goes away with the branch above. + ret = ddbc_bindings.DDBCSQLExecuteLegacy( + self.hstmt, + operation, + parameters, + parameters_type, + self.is_stmt_prepared, + effective_use_prepare, + encoding_settings, + ) # Check return code try: diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 2ba176d7..73eaab72 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -8,34 +8,24 @@ #include "connection/connection.h" #include "connection/connection_pool.h" #include "logger_bridge.hpp" +#include "param_detect.hpp" +#include "py_ref.hpp" +#include "py_type_cache.hpp" #include "utf_utils.h" - #include // std::min -#include #include #include // For std::memcpy #include -#include // std::setw, std::setfill #include #include // std::forward +#include // CPython datetime API (PyDateTime_IMPORT, PyDateTime_GET_*, etc.) //------------------------------------------------------------------------------------------------- // Macro definitions //------------------------------------------------------------------------------------------------- -// These constants are not exposed via sql.h, hence define them here -#define SQL_SS_TIME2 (-154) -#define SQL_SS_TIMESTAMPOFFSET (-155) -#define SQL_C_SS_TIME2 (0x4000) -#define SQL_C_SS_TIMESTAMPOFFSET (0x4001) -#define MAX_DIGITS_IN_NUMERIC 64 -#define SQL_MAX_NUMERIC_LEN 16 -#define SQL_SS_XML (-152) -#define SQL_SS_UDT (-151) -#define SQL_SS_VARIANT (-150) -#define SQL_CA_SS_VARIANT_TYPE (1215) #ifndef SQL_C_DATE #define SQL_C_DATE (9) #endif @@ -101,9 +91,6 @@ inline int EffectiveCharCtypeForFetch(int charCtype, const std::string& charEnco return charCtype; } -namespace PythonObjectCache { -py::object get_time_class(); -} //------------------------------------------------------------------------------------------------- //------------------------------------------------------------------------------------------------- @@ -113,159 +100,12 @@ py::object get_time_class(); // Uses printf-style formatting: LOG("Value: %d", x) -- __FILE__/__LINE__ // embedded in macro //------------------------------------------------------------------------------------------------- -namespace PythonObjectCache { -static py::object datetime_class; -static py::object date_class; -static py::object time_class; -static py::object decimal_class; -static py::object uuid_class; -static bool cache_initialized = false; - -void initialize() { - if (!cache_initialized) { - auto datetime_module = py::module_::import("datetime"); - datetime_class = datetime_module.attr("datetime"); - date_class = datetime_module.attr("date"); - time_class = datetime_module.attr("time"); - - auto decimal_module = py::module_::import("decimal"); - decimal_class = decimal_module.attr("Decimal"); - - auto uuid_module = py::module_::import("uuid"); - uuid_class = uuid_module.attr("UUID"); - - cache_initialized = true; - } -} - -py::object get_datetime_class() { - if (cache_initialized && datetime_class) { - return datetime_class; - } - return py::module_::import("datetime").attr("datetime"); -} - -py::object get_date_class() { - if (cache_initialized && date_class) { - return date_class; - } - return py::module_::import("datetime").attr("date"); -} - -py::object get_time_class() { - if (cache_initialized && time_class) { - return time_class; - } - return py::module_::import("datetime").attr("time"); -} - -py::object get_decimal_class() { - if (cache_initialized && decimal_class) { - return decimal_class; - } - return py::module_::import("decimal").attr("Decimal"); -} - -py::object get_uuid_class() { - if (cache_initialized && uuid_class) { - return uuid_class; - } - return py::module_::import("uuid").attr("UUID"); -} -} // namespace PythonObjectCache //------------------------------------------------------------------------------------------------- // Class definitions //------------------------------------------------------------------------------------------------- // Struct to hold parameter information for binding. Used by SQLBindParameter. -// This struct is shared between C++ & Python code. -// Suppress -Wattributes warning for ParamInfo struct -// The warning is triggered because pybind11 handles visibility attributes automatically, -// and having additional attributes on the struct can cause conflicts on Linux with GCC -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wattributes" -#endif -struct ParamInfo { - SQLSMALLINT inputOutputType; - SQLSMALLINT paramCType; - SQLSMALLINT paramSQLType; - SQLULEN columnSize; - SQLSMALLINT decimalDigits; - SQLLEN strLenOrInd = 0; // Required for DAE - bool isDAE = false; // Indicates if we need to stream - py::object dataPtr; -}; -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - -// Mirrors the SQL_NUMERIC_STRUCT. But redefined to replace val char array -// with std::string, because pybind doesn't allow binding char array. -// This struct is shared between C++ & Python code. -struct NumericData { - SQLCHAR precision; - SQLSCHAR scale; - SQLCHAR sign; // 1=pos, 0=neg - std::string val; // 123.45 -> 12345 - - NumericData() : precision(0), scale(0), sign(0), val(SQL_MAX_NUMERIC_LEN, '\0') {} - - NumericData(SQLCHAR precision, SQLSCHAR scale, SQLCHAR sign, const std::string& valueBytes) - : precision(precision), scale(scale), sign(sign), val(SQL_MAX_NUMERIC_LEN, '\0') { - if (valueBytes.size() > SQL_MAX_NUMERIC_LEN) { - throw std::runtime_error( - "NumericData valueBytes size exceeds SQL_MAX_NUMERIC_LEN (16)"); - } - // Copy binary data to buffer, remaining bytes stay zero-padded - std::memcpy(&val[0], valueBytes.data(), valueBytes.size()); - } -}; - -struct Int128_t { - uint64_t low; - int64_t high; - - Int128_t() : low(0), high(0) {} - Int128_t(uint64_t l, int64_t h) : low(l), high(h) {} - - Int128_t multiply_by_10() const { - // value * 10 = (value * 8) + (value * 2) - Int128_t shift3 = *this << 3; - Int128_t shift1 = *this << 1; - return shift3 + shift1; - } - - Int128_t operator<<(int shift) const { - // These would require special cases. We only shift by 1 and 3 for multiply_by_10. - assert(shift > 0); - assert(shift < 64); - uint64_t new_low = low << shift; - uint64_t new_high = (static_cast(high) << shift) | (low >> (64 - shift)); - return {new_low, static_cast(new_high)}; - } - - Int128_t operator+(const Int128_t& other) const { - uint64_t sum_low = low + other.low; - uint64_t carry = (sum_low < low) ? 1 : 0; - int64_t sum_high = high + other.high + carry; - return {sum_low, sum_high}; - } - - Int128_t operator+(uint64_t digit) const { - uint64_t sum_low = low + digit; - uint64_t carry = (sum_low < low) ? 1 : 0; - int64_t sum_high = high + carry; - return {sum_low, sum_high}; - } - - Int128_t operator-() const { - uint64_t new_low = ~low + 1; - uint64_t new_high = ~high + (new_low == 0 ? 1 : 0); - return {new_low, static_cast(new_high)}; - } -}; struct ArrowArrayPrivateData { std::unique_ptr valid; @@ -403,6 +243,7 @@ SQLDescribeParamFunc SQLDescribeParam_ptr = nullptr; namespace { + const char* GetSqlCTypeAsString(const SQLSMALLINT cType) { switch (cType) { STRINGIFY_FOR_CASE(SQL_C_CHAR); @@ -472,6 +313,22 @@ std::string DescribeChar(unsigned char ch) { } } + + + +template +// The callable hides whether the caller wraps SQLPutData with GIL management; chunk sizing stays shared. +static SQLRETURN stream_dae_chunks(const void* data, size_t total_bytes, PutDataFn put_data_fn) { + const char* bytes = static_cast(data); + for (size_t offset = 0; offset < total_bytes; offset += DAE_CHUNK_SIZE) { + size_t len = std::min(static_cast(DAE_CHUNK_SIZE), total_bytes - offset); + SQLRETURN rc = put_data_fn( + static_cast(const_cast(bytes + offset)), static_cast(len)); + if (!SQL_SUCCEEDED(rc)) return rc; + } + return SQL_SUCCESS; +} + // GH-610: Resolve SQL type for a NULL parameter using per-handle cache. // On cache miss, calls SQLDescribeParam and stores the result. static DescribedParamInfo ResolveNullParamType(SqlHandle& handle, SQLHANDLE hStmt, int paramIndex) { @@ -644,10 +501,12 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par std::string* strParam = AllocateParamBuffer(paramBuffers, encodedStr); - dataPtr = const_cast(static_cast(strParam->c_str())); - bufferLength = strParam->size() + 1; + dataPtr = const_cast(static_cast(strParam->data())); + bufferLength = strParam->size(); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); - *strLenOrIndPtr = SQL_NTS; + // Use explicit byte length instead of SQL_NTS so embedded NUL chars + // aren't treated as string terminators (e.g., "hello\x00world"). + *strLenOrIndPtr = static_cast(strParam->size()); } break; } @@ -712,7 +571,9 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par dataPtr = sqlwcharBuffer->data(); bufferLength = sqlwcharBuffer->size() * sizeof(SQLWCHAR); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); - *strLenOrIndPtr = SQL_NTS; + // Use explicit byte length instead of SQL_NTS so embedded NUL chars + // aren't treated as string terminators. + *strLenOrIndPtr = static_cast(sqlwcharBuffer->size() * sizeof(SQLWCHAR)); } break; } @@ -819,7 +680,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_TYPE_DATE: { - py::object dateType = PythonObjectCache::get_date_class(); + py::object dateType = PyTypeCache::get_date_class_obj(); if (!py::isinstance(param, dateType)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } @@ -839,7 +700,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_TYPE_TIME: { - py::object timeType = PythonObjectCache::get_time_class(); + py::object timeType = PyTypeCache::get_time_class_obj(); if (!py::isinstance(param, timeType)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } @@ -853,7 +714,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_SS_TIMESTAMPOFFSET: { - py::object datetimeType = PythonObjectCache::get_datetime_class(); + py::object datetimeType = PyTypeCache::get_datetime_class_obj(); if (!py::isinstance(param, datetimeType)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } @@ -905,7 +766,7 @@ SQLRETURN BindParameters(SqlHandle& handle, SQLHANDLE hStmt, const py::list& par break; } case SQL_C_TYPE_TIMESTAMP: { - py::object datetimeType = PythonObjectCache::get_datetime_class(); + py::object datetimeType = PyTypeCache::get_datetime_class_obj(); if (!py::isinstance(param, datetimeType)) { ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } @@ -1910,11 +1771,19 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, const std::u16string& cat return ret; } -// Executes the provided query. If the query is parametrized, it prepares the -// statement and binds the parameters. Otherwise, it executes the query -// directly. 'usePrepare' parameter can be used to disable the prepare step for -// queries that might already be prepared in a previous call. -SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const std::u16string& query, +// LEGACY — slated for removal in a future optimization round. +// +// Executes the provided query using a ParamInfo list that Python already built, +// rather than detecting parameter types in C++. Retained only for setinputsizes() +// callers, whose explicit type overrides the native path does not yet honour. +// Every parameter crosses the pybind11 boundary as a ParamInfo object here, which +// is the cost SQLExecute_wrap exists to avoid. Once setinputsizes is handled +// natively this function and its DDBCSQLExecuteLegacy binding both go away. +// +// If the query is parametrized, it prepares the statement and binds the +// parameters. Otherwise, it executes the query directly. 'usePrepare' can be used +// to disable the prepare step for queries already prepared in a previous call. +SQLRETURN SQLExecuteLegacy_wrap(const SqlHandlePtr statementHandle, const std::u16string& query, const py::list& params, std::vector& paramInfos, py::list& isStmtPrepared, const bool usePrepare, const py::dict& encodingSettings) { @@ -2007,7 +1876,6 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const std::u16stri if (!SQL_SUCCEEDED(rc)) { return rc; } - { // Release the GIL during the blocking SQLExecute network call. py::gil_scoped_release release; @@ -2040,89 +1908,66 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const std::u16stri if (!matchedInfo) { ThrowStdException("Unrecognized paramToken returned by SQLParamData"); } - const py::object& pyObj = matchedInfo->dataPtr; - if (pyObj.is_none()) { + PyObject* pyObj = matchedInfo->dataPtr.ptr(); + if (!pyObj || pyObj == Py_None) { putData(nullptr, 0); continue; } - if (py::isinstance(pyObj)) { + if (PyUnicode_Check(pyObj)) { if (matchedInfo->paramCType == SQL_C_WCHAR) { - std::u16string utf16 = pyObj.cast(); - size_t totalChars = utf16.size(); - const SQLWCHAR* dataPtr = reinterpretU16stringAsSqlWChar(utf16); - size_t offset = 0; - size_t chunkChars = DAE_CHUNK_SIZE / sizeof(SQLWCHAR); - while (offset < totalChars) { - size_t len = std::min(chunkChars, totalChars - offset); - size_t lenBytes = len * sizeof(SQLWCHAR); - if (lenBytes > - static_cast(std::numeric_limits::max())) { - ThrowStdException("Chunk size exceeds maximum " - "allowed by SQLLEN"); - } - rc = putData((SQLPOINTER)(dataPtr + offset), - static_cast(lenBytes)); - if (!SQL_SUCCEEDED(rc)) { - LOG("SQLExecute: SQLPutData failed for " - "SQL_C_WCHAR chunk - offset=%zu", - offset, totalChars, lenBytes, rc); - return rc; - } - offset += len; + std::u16string utf16 = + borrow(pyObj).cast(); + rc = stream_dae_chunks( + reinterpretU16stringAsSqlWChar(utf16), + utf16.size() * sizeof(SQLWCHAR), + putData); + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecute: SQLPutData failed for SQL_C_WCHAR DAE streaming"); + return rc; } } else if (matchedInfo->paramCType == SQL_C_CHAR) { // Encode the string using the specified encoding std::string encodedStr; try { - if (py::isinstance(pyObj)) { - py::object encoded = pyObj.attr("encode")(charEncoding, "strict"); - encodedStr = encoded.cast(); - LOG("SQLExecute: DAE SQL_C_CHAR - Encoded with '%s', %zu bytes", - charEncoding.c_str(), encodedStr.size()); - } else { - encodedStr = pyObj.cast(); - } + py::object encoded = borrow(pyObj) + .attr("encode")(charEncoding, "strict"); + encodedStr = encoded.cast(); + LOG("SQLExecute: DAE SQL_C_CHAR - Encoded with '%s', %zu bytes", + charEncoding.c_str(), encodedStr.size()); } catch (const py::error_already_set& e) { LOG_ERROR("SQLExecute: DAE SQL_C_CHAR - Failed to encode with '%s': %s", charEncoding.c_str(), e.what()); throw; } - size_t totalBytes = encodedStr.size(); - const char* dataPtr = encodedStr.data(); - size_t offset = 0; - size_t chunkBytes = DAE_CHUNK_SIZE; - while (offset < totalBytes) { - size_t len = std::min(chunkBytes, totalBytes - offset); - - rc = putData((SQLPOINTER)(dataPtr + offset), static_cast(len)); - if (!SQL_SUCCEEDED(rc)) { - LOG("SQLExecute: SQLPutData failed for " - "SQL_C_CHAR chunk - offset=%zu", - offset, totalBytes, len, rc); - return rc; - } - offset += len; + rc = stream_dae_chunks(encodedStr.data(), encodedStr.size(), putData); + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecute: SQLPutData failed for SQL_C_CHAR DAE streaming"); + return rc; } } else { ThrowStdException("Unsupported C type for str in DAE"); } - } else if (py::isinstance(pyObj) || - py::isinstance(pyObj)) { - py::bytes b = pyObj.cast(); - std::string s = b; - const char* dataPtr = s.data(); - size_t totalBytes = s.size(); - const size_t chunkSize = DAE_CHUNK_SIZE; - for (size_t offset = 0; offset < totalBytes; offset += chunkSize) { - size_t len = std::min(chunkSize, totalBytes - offset); - rc = putData((SQLPOINTER)(dataPtr + offset), static_cast(len)); - if (!SQL_SUCCEEDED(rc)) { - LOG("SQLExecute: SQLPutData failed for " - "binary/bytes chunk - offset=%zu", - offset, totalBytes, len, rc); - return rc; - } + } else if (PyBytes_Check(pyObj) || PyByteArray_Check(pyObj)) { + const char* dataPtr = nullptr; + size_t totalBytes = 0; + std::string bytesStorage; // lifetime must span the loop + if (PyBytes_Check(pyObj)) { + bytesStorage = borrow(pyObj); + dataPtr = bytesStorage.data(); + totalBytes = bytesStorage.size(); + } else { + // bytearray is mutable — copy to stable buffer before streaming + bytesStorage.assign(PyByteArray_AS_STRING(pyObj), + static_cast(PyByteArray_GET_SIZE(pyObj))); + dataPtr = bytesStorage.data(); + totalBytes = bytesStorage.size(); + } + + rc = stream_dae_chunks(dataPtr, totalBytes, putData); + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecute: SQLPutData failed for binary/bytes DAE streaming"); + return rc; } } else { ThrowStdException("DAE only supported for str or bytes"); @@ -2143,14 +1988,188 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const std::u16stri return rc; } - // Unbind the bound buffers for all parameters coz the buffers' memory - // will be freed when this function exits (parambuffers goes out of - // scope) - rc = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + // Unbind parameter buffers before they go out of scope. + // Not called on error paths — diagnostics must remain readable. + SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); return rc; } } +// --------------------------------------------------------------------------- +// SQLExecute_wrap — single C++ pipeline: DetectParamTypes → BindParameters → SQLExecute +// No ParamInfo objects cross the pybind11 boundary. +// +// Honors use_prepare: when true, uses SQLPrepare + SQLExecute (benefiting from +// plan reuse). When false but already prepared, reuses the existing plan. +// When false and not prepared, throws (matching slow path behavior). +// --------------------------------------------------------------------------- +SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, + const std::u16string& query, + py::list params, + py::list is_stmt_prepared, + bool use_prepare, + const py::dict& encoding_settings) { + if (!statementHandle || !statementHandle->get()) { + return SQL_INVALID_HANDLE; + } + + SQLHANDLE hStmt = statementHandle->get(); + + // Configure forward-only / read-only cursor (matches slow path semantics). + if (SQLSetStmtAttr_ptr) { + SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_CURSOR_TYPE, + (SQLPOINTER)SQL_CURSOR_FORWARD_ONLY, 0); + SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_CONCURRENCY, + (SQLPOINTER)SQL_CONCUR_READ_ONLY, 0); + } + + // The encoding-settings dict has the form {"encoding": str, "ctype": int}. + // Note: the Python layer's SQL_C_CHAR constant is numerically -8, the same + // as ODBC's SQL_C_WCHAR. As a result, the only path that genuinely uses + // byte-level character encoding is when the user explicitly opts in via + // setencoding(..., ctype=mssql_python.SQL_CHAR) (which sends ctype=1, the + // real ODBC SQL_CHAR). We default to utf-8 and only honor the dict's + // encoding when ctype == 1 (real ODBC SQL_CHAR). Otherwise the user's + // "encoding" value is meant for the wide-char path and we leave it alone. + std::string charEncoding = "utf-8"; + if (encoding_settings.contains("ctype") && encoding_settings.contains("encoding")) { + int ctype = encoding_settings["ctype"].cast(); + if (ctype == SQL_C_CHAR /* real ODBC value: 1 */) { + charEncoding = encoding_settings["encoding"].cast(); + } + } + + // The cursor.py caller always passes a fresh `list(actual_params)` so this + // function is free to mutate slots in place. Even so, every site below uses + // PyList_SetItem (which decrefs the old slot before stealing the new ref), + // so the function is safe regardless of who owns the list. + + // Run DetectParamTypes BEFORE SQLPrepare so that type-detection errors + // (unsupported type, NaN Decimal, precision overflow) don't leave the + // cursor in a half-prepared state. + std::vector paramInfos = DetectParamTypes(params.ptr()); + + RETCODE rc; + bool already_prepared = is_stmt_prepared[0].cast(); + + // Honor use_prepare flag (matching slow path behavior): + // - use_prepare=true: prepare now (or reuse if same SQL already prepared) + // - use_prepare=false + already prepared: reuse existing plan + // - use_prepare=false + not prepared: error (cannot execute unprepared) + if (!already_prepared) { + if (use_prepare) { + SQLWCHAR* queryPtr = reinterpretU16stringAsSqlWChar(query); + { + py::gil_scoped_release release; + rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); + } + if (!SQL_SUCCEEDED(rc)) return rc; + statementHandle->clearDescribeCache(); + is_stmt_prepared[0] = py::bool_(true); + } else { + ThrowStdException("Cannot execute unprepared statement"); + } + } + + std::vector> paramBuffers; + rc = BindParameters(*statementHandle, hStmt, params, paramInfos, paramBuffers, charEncoding); + if (!SQL_SUCCEEDED(rc)) return rc; + + { + py::gil_scoped_release release; + rc = SQLExecute_ptr(hStmt); + } + + // DAE (Data-At-Execution) loop: when BindParameters marks a param as DAE + // (large str/bytes/binary), SQLExecute returns SQL_NEED_DATA. We must + // stream the data via SQLParamData/SQLPutData before execution completes. + // GIL is released around each ODBC call to match slow-path concurrency. + if (rc == SQL_NEED_DATA) { + SQLPOINTER paramToken = nullptr; + auto putData = [&](SQLPOINTER data, SQLLEN len) { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, data, len); + }; + while (true) { + { + py::gil_scoped_release release; + rc = SQLParamData_ptr(hStmt, ¶mToken); + } + if (rc != SQL_NEED_DATA) break; + + const ParamInfo* matchedInfo = nullptr; + for (auto& info : paramInfos) { + if (reinterpret_cast(const_cast(&info)) == paramToken) { + matchedInfo = &info; + break; + } + } + if (!matchedInfo) { + ThrowStdException("SQLExecute: unrecognized paramToken from SQLParamData"); + } + PyObject* pyObj = matchedInfo->dataPtr.ptr(); + if (!pyObj || pyObj == Py_None) { + py::gil_scoped_release release; + SQLPutData_ptr(hStmt, nullptr, 0); + continue; + } + + if (PyUnicode_Check(pyObj)) { + if (matchedInfo->paramCType == SQL_C_WCHAR) { + std::u16string u16 = + borrow(pyObj).cast(); + rc = stream_dae_chunks( + reinterpretU16stringAsSqlWChar(u16), + u16.size() * sizeof(SQLWCHAR), + putData); + if (!SQL_SUCCEEDED(rc)) return rc; + } else if (matchedInfo->paramCType == SQL_C_CHAR) { + std::string encodedStr; + py::object encoded = borrow(pyObj) + .attr("encode")(charEncoding, "strict"); + encodedStr = encoded.cast(); + rc = stream_dae_chunks(encodedStr.data(), encodedStr.size(), putData); + if (!SQL_SUCCEEDED(rc)) return rc; + } else { + ThrowStdException("SQLExecute: unsupported C type for str in DAE"); + } + } else if (PyBytes_Check(pyObj) || PyByteArray_Check(pyObj)) { + // Handle bytes and bytearray separately — pybind11's bytes + // caster does not safely convert bytearray. + const char* dataPtr = nullptr; + size_t totalBytes = 0; + std::string bytesStorage; // lifetime must span the loop + + if (PyBytes_Check(pyObj)) { + bytesStorage = borrow(pyObj); + dataPtr = bytesStorage.data(); + totalBytes = bytesStorage.size(); + } else { + // bytearray is mutable — copy to stable buffer before streaming + bytesStorage.assign(PyByteArray_AS_STRING(pyObj), + static_cast(PyByteArray_GET_SIZE(pyObj))); + dataPtr = bytesStorage.data(); + totalBytes = bytesStorage.size(); + } + + rc = stream_dae_chunks(dataPtr, totalBytes, putData); + if (!SQL_SUCCEEDED(rc)) return rc; + } else { + ThrowStdException("SQLExecute: DAE only supported for str or bytes"); + } + } + if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) return rc; + } + + if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) return rc; + + // Unbind parameter buffers before they go out of scope. + // Not called on error paths — diagnostics must remain readable. + SQLRETURN exec_rc = rc; + SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS); + return exec_rc; +} + SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, @@ -2575,7 +2594,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); - py::object datetimeType = PythonObjectCache::get_datetime_class(); + py::object datetimeType = PyTypeCache::get_datetime_class_obj(); for (size_t i = 0; i < paramSetSize; ++i) { const py::handle& param = columnValues[i]; @@ -2690,7 +2709,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& // Get cached UUID class from module-level helper // This avoids static object destruction issues during // Python finalization - py::object uuid_class = PythonObjectCache::get_uuid_class(); + py::object uuid_class = PyTypeCache::get_uuid_class_obj(); // Get cached UUID class for (size_t i = 0; i < paramSetSize; ++i) { @@ -3657,7 +3676,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p // parsing The decimal separator only affects display // formatting, not parsing py::object decimalObj = - PythonObjectCache::get_decimal_class()(py::str(cnum, safeLen)); + PyTypeCache::get_decimal_class_obj()(py::str(cnum, safeLen)); row.append(decimalObj); } catch (const py::error_already_set& e) { // If conversion fails, append None @@ -3707,7 +3726,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ret = SQLGetData_ptr(hStmt, i, SQL_C_TYPE_DATE, &dateValue, sizeof(dateValue), NULL); if (SQL_SUCCEEDED(ret)) { - row.append(PythonObjectCache::get_date_class()(dateValue.year, dateValue.month, + row.append(PyTypeCache::get_date_class_obj()(dateValue.year, dateValue.month, dateValue.day)); } else { row.append(py::none()); @@ -3720,7 +3739,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p SQLLEN indicator = 0; ret = SQLGetData_ptr(hStmt, i, SQL_C_SS_TIME2, &t2, sizeof(t2), &indicator); if (SQL_SUCCEEDED(ret) && indicator != SQL_NULL_DATA) { - row.append(PythonObjectCache::get_time_class()( + row.append(PyTypeCache::get_time_class_obj()( t2.hour, t2.minute, t2.second, t2.fraction / 1000)); // ns to µs } else { if (!SQL_SUCCEEDED(ret)) { @@ -3739,7 +3758,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ret = SQLGetData_ptr(hStmt, i, SQL_C_TYPE_TIMESTAMP, ×tampValue, sizeof(timestampValue), NULL); if (SQL_SUCCEEDED(ret)) { - row.append(PythonObjectCache::get_datetime_class()( + row.append(PyTypeCache::get_datetime_class_obj()( timestampValue.year, timestampValue.month, timestampValue.day, timestampValue.hour, timestampValue.minute, timestampValue.second, timestampValue.fraction / 1000 // Convert back ns to µs @@ -3779,7 +3798,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p py::object datetime_module = py::module_::import("datetime"); py::object tzinfo = datetime_module.attr("timezone")( datetime_module.attr("timedelta")(py::arg("minutes") = totalMinutes)); - py::object py_dt = PythonObjectCache::get_datetime_class()( + py::object py_dt = PyTypeCache::get_datetime_class_obj()( dtoValue.year, dtoValue.month, dtoValue.day, dtoValue.hour, dtoValue.minute, dtoValue.second, microseconds, tzinfo); row.append(py_dt); @@ -3886,7 +3905,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p py::bytes py_guid_bytes(guid_bytes.data(), guid_bytes.size()); py::object uuid_obj = - PythonObjectCache::get_uuid_class()(py::arg("bytes") = py_guid_bytes); + PyTypeCache::get_uuid_class_obj()(py::arg("bytes") = py_guid_bytes); row.append(uuid_obj); } else if (indicator == SQL_NULL_DATA) { row.append(py::none()); @@ -4349,7 +4368,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum // parsing The decimal separator only affects display // formatting, not parsing PyObject* decimalObj = - PythonObjectCache::get_decimal_class()(py::str(rawData, decimalDataLen)) + PyTypeCache::get_decimal_class_obj()(py::str(rawData, decimalDataLen)) .release() .ptr(); PyList_SET_ITEM(row, col - 1, decimalObj); @@ -4366,7 +4385,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum case SQL_TYPE_TIMESTAMP: case SQL_DATETIME: { const SQL_TIMESTAMP_STRUCT& ts = buffers.timestampBuffers[col - 1][i]; - PyObject* datetimeObj = PythonObjectCache::get_datetime_class()( + PyObject* datetimeObj = PyTypeCache::get_datetime_class_obj()( ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second, ts.fraction / 1000) .release() @@ -4376,7 +4395,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum } case SQL_TYPE_DATE: { PyObject* dateObj = - PythonObjectCache::get_date_class()(buffers.dateBuffers[col - 1][i].year, + PyTypeCache::get_date_class_obj()(buffers.dateBuffers[col - 1][i].year, buffers.dateBuffers[col - 1][i].month, buffers.dateBuffers[col - 1][i].day) .release() @@ -4387,7 +4406,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum case SQL_SS_TIME2: { const SQL_SS_TIME2_STRUCT& t2 = buffers.timeBuffers[col - 1][i]; PyObject* timeObj = - PythonObjectCache::get_time_class()(t2.hour, t2.minute, t2.second, + PyTypeCache::get_time_class_obj()(t2.hour, t2.minute, t2.second, t2.fraction / 1000) // ns to µs .release() .ptr(); @@ -4403,7 +4422,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum py::object datetime_module = py::module_::import("datetime"); py::object tzinfo = datetime_module.attr("timezone")( datetime_module.attr("timedelta")(py::arg("minutes") = totalMinutes)); - py::object py_dt = PythonObjectCache::get_datetime_class()( + py::object py_dt = PyTypeCache::get_datetime_class_obj()( dtoValue.year, dtoValue.month, dtoValue.day, dtoValue.hour, dtoValue.minute, dtoValue.second, dtoValue.fraction / 1000, // ns → µs @@ -4437,7 +4456,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum py::bytes py_guid_bytes(reinterpret_cast(reordered), 16); py::dict kwargs; kwargs["bytes"] = py_guid_bytes; - py::object uuid_obj = PythonObjectCache::get_uuid_class()(**kwargs); + py::object uuid_obj = PyTypeCache::get_uuid_class_obj()(**kwargs); PyList_SET_ITEM(row, col - 1, uuid_obj.release().ptr()); break; } @@ -5966,7 +5985,7 @@ void DDBCSetDecimalSeparator(const std::string& separator) { PYBIND11_MODULE(ddbc_bindings, m) { m.doc() = "msodbcsql driver api bindings for Python"; - PythonObjectCache::initialize(); + PyTypeCache::initialize(); // Add architecture information as module attribute m.attr("__architecture__") = ARCHITECTURE; @@ -5989,7 +6008,15 @@ PYBIND11_MODULE(ddbc_bindings, m) { .def_readwrite("columnSize", &ParamInfo::columnSize) .def_readwrite("decimalDigits", &ParamInfo::decimalDigits) .def_readwrite("strLenOrInd", &ParamInfo::strLenOrInd) - .def_readwrite("dataPtr", &ParamInfo::dataPtr) + .def_property( + "dataPtr", + [](const ParamInfo& info) -> py::object { + if (!info.dataPtr) { + return py::none(); + } + return info.dataPtr; + }, + [](ParamInfo& info, py::object obj) { info.dataPtr = std::move(obj); }) .def_readwrite("isDAE", &ParamInfo::isDAE); // Define numeric data class @@ -6038,9 +6065,17 @@ PYBIND11_MODULE(ddbc_bindings, m) { manager.closePools(); }, "Disable global connection pooling and close all pools"); m.def("DDBCSQLExecDirect", &SQLExecDirect_wrap, "Execute a SQL query directly"); - m.def("DDBCSQLExecute", &SQLExecute_wrap, "Prepare and execute T-SQL statements", + // LEGACY — to be removed once setinputsizes overrides are handled natively. + m.def("DDBCSQLExecuteLegacy", &SQLExecuteLegacy_wrap, + "Legacy path (slated for removal): accepts pre-built ParamInfo from Python, " + "used only when setinputsizes overrides are present", py::arg("statementHandle"), py::arg("query"), py::arg("params"), py::arg("paramInfos"), py::arg("isStmtPrepared"), py::arg("usePrepare"), py::arg("encodingSettings")); + // Standard path — what cursor.execute() uses unless setinputsizes is active. + m.def("DDBCSQLExecute", &SQLExecute_wrap, + "Standard path: DetectParamTypes + BindParameters + SQLExecute all in C++", + py::arg("statementHandle"), py::arg("query"), py::arg("params"), + py::arg("isStmtPrepared"), py::arg("usePrepare"), py::arg("encodingSettings")); m.def("SQLExecuteMany", &SQLExecuteMany_wrap, "Execute statement with multiple parameter sets", py::arg("statementHandle"), py::arg("query"), py::arg("columnwise_params"), py::arg("paramInfos"), py::arg("paramSetSize"), py::arg("encodingSettings")); diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index d6e4acca..684c68a2 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -32,6 +32,25 @@ using py::literals::operator""_a; #include #include +//------------------------------------------------------------------------------------------------- +// SQL Server specific ODBC constants +// +// These are not exposed via sql.h / sqlext.h, so they are defined here. They live in this shared +// header rather than in a single .cpp because both the parameter-detection path +// (param_detect.hpp) and the fetch paths in ddbc_bindings.cpp need them. +//------------------------------------------------------------------------------------------------- + +#define SQL_SS_TIME2 (-154) +#define SQL_SS_TIMESTAMPOFFSET (-155) +#define SQL_C_SS_TIME2 (0x4000) +#define SQL_C_SS_TIMESTAMPOFFSET (0x4001) +#define MAX_DIGITS_IN_NUMERIC 64 +#define SQL_MAX_NUMERIC_LEN 16 +#define SQL_SS_XML (-152) +#define SQL_SS_UDT (-151) +#define SQL_SS_VARIANT (-150) +#define SQL_CA_SS_VARIANT_TYPE (1215) + // Include logger bridge for LOG macros #include "logger_bridge.hpp" diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp new file mode 100644 index 00000000..6407b822 --- /dev/null +++ b/mssql_python/pybind/param_detect.hpp @@ -0,0 +1,658 @@ +// param_detect.hpp — Python parameter type detection for the primary execute path. +// +// Owns the first stage of the native execute pipeline: +// +// DetectParamTypes -> BindParameters -> SQLExecute +// (this file) (ddbc_bindings.cpp) +// +// DetectParamTypes inspects each Python parameter, decides the ODBC C type / SQL type +// / column size to bind it as, and returns a ParamInfo per parameter. It also carries +// the ParamInfo / NumericData / Int128_t types those results are expressed in, and +// build_numeric_data, which converts a Python Decimal into the SQL_NUMERIC_STRUCT +// byte layout. +// +// Header-only, and deliberately so. The build compiles with -O3 but without LTO, so a +// .cpp boundary would also be an inlining boundary: the small helpers here are called +// once per parameter per execute, and moving them out of the caller's translation unit +// would turn inlined code into real calls on the hot path. Defining them inline in a +// header keeps them in whichever translation unit uses them. If LTO is enabled later +// this can become a normal .cpp. + +#pragma once + +#include "ddbc_bindings.h" // ParamInfo consumers, SQL Server ODBC constants +#include "py_ref.hpp" // steal / borrow +#include "py_type_cache.hpp" // PyTypeCache::get_*_class + +#include +#include // CPython datetime API (PyDateTime_Check, PyDateTime_GET_*, etc.) + +#include // std::min +#include +#include // snprintf +#include // std::memcpy +#include +#include + +//------------------------------------------------------------------------------------------------- +// Parameter description types +//------------------------------------------------------------------------------------------------- + +// This struct is shared between C++ & Python code. +// Suppress -Wattributes warning for ParamInfo struct +// The warning is triggered because pybind11 handles visibility attributes automatically, +// and having additional attributes on the struct can cause conflicts on Linux with GCC +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wattributes" +#endif +struct ParamInfo { + SQLSMALLINT inputOutputType = SQL_PARAM_INPUT; + SQLSMALLINT paramCType = SQL_C_DEFAULT; + SQLSMALLINT paramSQLType = SQL_UNKNOWN_TYPE; + SQLULEN columnSize = 0; + SQLSMALLINT decimalDigits = 0; + SQLLEN strLenOrInd = 0; // Required for DAE + bool isDAE = false; // Indicates if we need to stream + // Strong reference to the Python object for DAE (data-at-execution) streaming. + // py::object owns the refcount, so the compiler-generated destructor, copy and + // move operations are all correct and this struct needs no rule-of-five. + py::object dataPtr; + Py_ssize_t utf16Len = 0; // UTF-16 code unit count for string params +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +// Mirrors the SQL_NUMERIC_STRUCT. But redefined to replace val char array +// with std::string, because pybind doesn't allow binding char array. +// This struct is shared between C++ & Python code. +struct NumericData { + SQLCHAR precision; + SQLSCHAR scale; + SQLCHAR sign; // 1=pos, 0=neg + std::string val; // 123.45 -> 12345 + + NumericData() : precision(0), scale(0), sign(0), val(SQL_MAX_NUMERIC_LEN, '\0') {} + + NumericData(SQLCHAR precision, SQLSCHAR scale, SQLCHAR sign, const std::string& valueBytes) + : precision(precision), scale(scale), sign(sign), val(SQL_MAX_NUMERIC_LEN, '\0') { + if (valueBytes.size() > SQL_MAX_NUMERIC_LEN) { + throw std::runtime_error( + "NumericData valueBytes size exceeds SQL_MAX_NUMERIC_LEN (16)"); + } + // Copy binary data to buffer, remaining bytes stay zero-padded + std::memcpy(&val[0], valueBytes.data(), valueBytes.size()); + } +}; + +struct Int128_t { + uint64_t low; + int64_t high; + + Int128_t() : low(0), high(0) {} + Int128_t(uint64_t l, int64_t h) : low(l), high(h) {} + + Int128_t multiply_by_10() const { + // value * 10 = (value * 8) + (value * 2) + Int128_t shift3 = *this << 3; + Int128_t shift1 = *this << 1; + return shift3 + shift1; + } + + Int128_t operator<<(int shift) const { + // These would require special cases. We only shift by 1 and 3 for multiply_by_10. + assert(shift > 0); + assert(shift < 64); + uint64_t new_low = low << shift; + uint64_t new_high = (static_cast(high) << shift) | (low >> (64 - shift)); + return {new_low, static_cast(new_high)}; + } + + Int128_t operator+(const Int128_t& other) const { + uint64_t sum_low = low + other.low; + uint64_t carry = (sum_low < low) ? 1 : 0; + int64_t sum_high = high + other.high + carry; + return {sum_low, sum_high}; + } + + Int128_t operator+(uint64_t digit) const { + uint64_t sum_low = low + digit; + uint64_t carry = (sum_low < low) ? 1 : 0; + int64_t sum_high = high + carry; + return {sum_low, sum_high}; + } + + Int128_t operator-() const { + uint64_t new_low = ~low + 1; + uint64_t new_high = ~high + (new_low == 0 ? 1 : 0); + return {new_low, static_cast(new_high)}; + } +}; + +// --------------------------------------------------------------------------- +// Constants for DetectParamTypes +// --------------------------------------------------------------------------- + +// Strings longer than this use data-at-execution (DAE) streaming +inline constexpr int MAX_INLINE_CHAR = 4000; + +// Binary data longer than this uses DAE streaming (SQL Server max for non-MAX types) +inline constexpr int MAX_INLINE_BINARY = 8000; + +// SQL Server maximum numeric precision +inline constexpr int MAX_NUMERIC_PRECISION = 38; + +// C type used to bind narrow (ASCII) text: SQL_C_WCHAR on every platform. +// +// The legacy Python path binds text with its own SQL_C_CHAR constant, which is +// numerically -8 — that is ODBC's SQL_C_WCHAR, not SQL_C_CHAR (1). So the legacy +// path has always bound text wide, on every platform. unixODBC also requires wide +// chars for text on Linux/macOS, so those platforms agreed already; Windows was +// the only one resolving this to a real SQL_C_CHAR and binding narrow, which made +// it diverge from both the legacy path and the other platforms in C type and in +// the driver-side encoding path it took. Bind wide everywhere so all four +// combinations agree. +inline constexpr SQLSMALLINT PARAM_C_TYPE_TEXT = SQL_C_WCHAR; + +// Forward declare NumericData helper used by decimal path +inline NumericData build_numeric_data(PyObject* as_tuple, PyObject* digits, int exponent); + +// True if the ready unicode string starts with the given ASCII literal, matching +// str.startswith semantics across every storage kind (UCS1/2/4). PyUnicode_READ +// yields the code point at each index regardless of kind, so a WKT prefix is +// detected even when a later non-ASCII char forces the string into a wider kind. +// The legacy path uses str.startswith, which is kind-independent, so native must +// be too for parity. +inline bool StartsWithAscii(unsigned int kind, const void* data, Py_ssize_t length, + const char* prefix, Py_ssize_t prefixLen) { + if (length < prefixLen) { + return false; + } + for (Py_ssize_t j = 0; j < prefixLen; ++j) { + if (PyUnicode_READ(kind, data, j) != static_cast(static_cast(prefix[j]))) { + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- +// DetectParamTypes: Raw CPython parameter type detection for the primary execute path. +// +// Design decisions: +// 1. Operates on a COPY of the user's param list (cursor.py does list(actual_params)). +// We mutate it in-place (PyList_SetItem) for types that need pre-processing +// (time→isoformat string, Decimal→NumericData, UUID→bytes_le). +// 2. Uses CPython macros (PyLong_Check, PyDateTime_Check, etc.) instead of pybind11's +// py::isinstance<> for ~3x faster type checks (direct struct field test vs virtual call). +// 3. Integer range detection uses constants — these match the SQL Server +// storage engine's range exactly (TINYINT: 0-255, SMALLINT: -32768..32767, etc.) +// 4. String handling inspects UCS kind directly for O(1) ASCII detection rather than +// scanning content — critical for bulk insert scenarios with thousands of params. +// 5. MONEY/SMALLMONEY uses exact Decimal comparison (PyObject_RichCompareBool) to avoid +// double-precision boundary errors (e.g., 214748.3647 would round incorrectly as double). +// --------------------------------------------------------------------------- +// +// ORDERING MATTERS: +// - bool before int (bool is a subclass of int in Python) +// - datetime before date (datetime is a subclass of date) +// +// Takes a raw PyObject* (must be a list). Caller guarantees it's a fresh copy +// (cursor.py does list(actual_params)), so in-place mutation via PyList_SetItem is safe. +inline std::vector DetectParamTypes(PyObject* params) { + PyTypeCache::initialize(); + + const Py_ssize_t n = PyList_GET_SIZE(params); + std::vector infos(n); + + PyObject* decimal_type = PyTypeCache::get_decimal_class(); + PyObject* uuid_type = PyTypeCache::get_uuid_class(); + + for (Py_ssize_t i = 0; i < n; ++i) { + ParamInfo& info = infos[i]; + info.inputOutputType = SQL_PARAM_INPUT; + info.isDAE = false; + + PyObject* obj = PyList_GET_ITEM(params, i); + + // --- None --- + if (obj == Py_None) { + info.paramSQLType = SQL_UNKNOWN_TYPE; + info.paramCType = SQL_C_DEFAULT; + info.columnSize = 1; + info.decimalDigits = 0; + continue; + } + + // bool must be checked before int: in CPython, PyBool_Type is a subclass of + // PyLong_Type, so PyLong_Check(True) returns 1. If we hit the int branch first, + // True→1 instead of BIT. + if (PyBool_Check(obj)) { + info.paramSQLType = SQL_BIT; + info.paramCType = SQL_C_BIT; + info.columnSize = 1; + info.decimalDigits = 0; + continue; + } + + // --- int (allow subclasses, but bool was already caught above) --- + if (PyLong_Check(obj)) { + int overflow = 0; + int64_t val = PyLong_AsLongLongAndOverflow(obj, &overflow); + if (overflow == 0 && !PyErr_Occurred()) { + if (val >= 0 && val <= UINT8_MAX) { + info.paramSQLType = SQL_TINYINT; + info.paramCType = SQL_C_TINYINT; + info.columnSize = 3; + } else if (val >= INT16_MIN && val <= INT16_MAX) { + info.paramSQLType = SQL_SMALLINT; + info.paramCType = SQL_C_SHORT; + info.columnSize = 5; + } else if (val >= INT32_MIN && val <= INT32_MAX) { + info.paramSQLType = SQL_INTEGER; + info.paramCType = SQL_C_LONG; + info.columnSize = 10; + } else { + info.paramSQLType = SQL_BIGINT; + info.paramCType = SQL_C_SBIGINT; + info.columnSize = 19; + } + } else { + PyErr_Clear(); + info.paramSQLType = SQL_BIGINT; + info.paramCType = SQL_C_SBIGINT; + info.columnSize = 19; + } + info.decimalDigits = 0; + continue; + } + + // --- float (allow subclasses) --- + if (PyFloat_Check(obj)) { + info.paramSQLType = SQL_DOUBLE; + info.paramCType = SQL_C_DOUBLE; + info.columnSize = 15; + info.decimalDigits = 0; + continue; + } + + // --- str (allow subclasses) --- + if (PyUnicode_Check(obj)) { + Py_ssize_t length = PyUnicode_GET_LENGTH(obj); + unsigned int kind = PyUnicode_KIND(obj); + const void* udata = PyUnicode_DATA(obj); + + Py_ssize_t utf16_len; + if (kind <= PyUnicode_2BYTE_KIND) { + utf16_len = length; + } else { + utf16_len = 0; + const Py_UCS4* data = PyUnicode_4BYTE_DATA(obj); + for (Py_ssize_t j = 0; j < length; ++j) { + utf16_len += (data[j] > 0xFFFF) ? 2 : 1; + } + } + + // Detect whether the string needs wide-char (NVARCHAR) or narrow (VARCHAR) binding. + // PyUnicode_IS_COMPACT_ASCII is a struct field check (O(1)), not a content scan. + // UCS-1 strings with max_char > 127 contain Latin-1 chars → need NVARCHAR. + bool is_unicode = + (kind > PyUnicode_1BYTE_KIND) || + (PyUnicode_IS_COMPACT_ASCII(obj) == 0 && kind == PyUnicode_1BYTE_KIND && + PyUnicode_MAX_CHAR_VALUE(obj) > 127); + + // Geometry WKT (POINT / LINESTRING / POLYGON) always binds as NVARCHAR, so the + // SQL type is stable regardless of the ASCII/Latin-1 heuristic above and matches + // the small-geometry case. The prefix is checked kind-agnostically (matching the + // legacy str.startswith), and this runs BEFORE the length/DAE decision so a large + // polygon keeps the SAME wide type while still taking the DAE path below. + // + // NB: we deliberately do NOT copy the legacy path's exact tuple here. The legacy + // _map_sql_type returns NVARCHAR with columnSize == len and DAE=false even for a + // 7790-char polygon, which is unbindable (SQLBindParameter rejects a non-MAX + // NVARCHAR precision > 4000 with "Invalid precision value"). Folding geometry into + // is_unicode instead keeps geometry on NVARCHAR for both size regimes and lets the + // length gate stream large values via DAE, which actually binds. + if (StartsWithAscii(kind, udata, length, "POINT", 5) || + StartsWithAscii(kind, udata, length, "LINESTRING", 10) || + StartsWithAscii(kind, udata, length, "POLYGON", 7)) { + is_unicode = true; + } + + if (utf16_len > MAX_INLINE_CHAR) { + // Strings > 4000 UTF-16 code units exceed SQL Server's inline NVARCHAR(MAX) + // threshold. Switch to data-at-execution (DAE) streaming: ODBC driver pulls + // data in chunks via SQLPutData, avoiding a single massive buffer allocation. + // DAE path: match slow-path types exactly. + // Non-unicode (ASCII) → SQL_VARCHAR + PARAM_C_TYPE_TEXT, which is + // SQL_C_WCHAR and matches the slow path's SQL_C_CHAR (numerically + // -8 == SQL_C_WCHAR — a long-standing alias in the Python layer). + // Unicode → SQL_WVARCHAR + SQL_C_WCHAR (wide-char streaming) + info.isDAE = true; + info.columnSize = 0; + info.utf16Len = utf16_len; + info.dataPtr = borrow(obj); + info.paramSQLType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; + info.paramCType = is_unicode ? SQL_C_WCHAR : PARAM_C_TYPE_TEXT; + } else { + info.columnSize = is_unicode ? utf16_len : length; + info.paramSQLType = is_unicode ? SQL_WVARCHAR : SQL_VARCHAR; + info.paramCType = is_unicode ? SQL_C_WCHAR : PARAM_C_TYPE_TEXT; + } + info.decimalDigits = 0; + continue; + } + + // --- bytes / bytearray (allow subclasses) --- + if (PyBytes_Check(obj) || PyByteArray_Check(obj)) { + Py_ssize_t length = PyBytes_Check(obj) ? PyBytes_Size(obj) : PyByteArray_Size(obj); + info.paramSQLType = SQL_VARBINARY; + info.paramCType = SQL_C_BINARY; + info.decimalDigits = 0; + if (length > MAX_INLINE_BINARY) { + info.isDAE = true; + info.columnSize = 0; + info.dataPtr = borrow(obj); + } else { + info.columnSize = std::max(length, 1); + } + continue; + } + + // --- datetime (must check before date, since datetime is subclass of date) --- + if (PyDateTime_Check(obj)) { + py::object tzinfo = steal(PyObject_GetAttrString(obj, "tzinfo")); + if (!tzinfo) throw py::error_already_set(); + bool has_tz = (tzinfo.ptr() != Py_None); + if (has_tz) { + info.paramSQLType = SQL_SS_TIMESTAMPOFFSET; + info.paramCType = SQL_C_SS_TIMESTAMPOFFSET; + info.columnSize = 34; + info.decimalDigits = 7; + } else { + info.paramSQLType = SQL_TYPE_TIMESTAMP; + info.paramCType = SQL_C_TYPE_TIMESTAMP; + info.columnSize = 26; + info.decimalDigits = 6; + } + continue; + } + + // --- date --- + if (PyDate_Check(obj)) { + info.paramSQLType = SQL_TYPE_DATE; + info.paramCType = SQL_C_TYPE_DATE; + info.columnSize = 10; + info.decimalDigits = 0; + continue; + } + + // --- time (normalized to string for binding) --- + if (PyTime_Check(obj)) { + info.paramSQLType = SQL_TYPE_TIME; + info.paramCType = + PARAM_C_TYPE_TEXT; // matches slow path (its SQL_C_CHAR is -8 = SQL_C_WCHAR) + info.columnSize = 16; + info.decimalDigits = 6; + // Delegate to isoformat rather than formatting the raw H/M/S/us fields by hand. + // Hand-formatting silently drops tzinfo (an aware time rendered as + // "01:02:03.000004+05:30" became "01:02:03.000004") and ignores isoformat + // overrides on time subclasses. The legacy path calls + // isoformat(timespec="microseconds") via _normalize_time_param in cursor.py, + // so calling the same method is what keeps the two paths in agreement. + py::object time_obj = steal(PyObject_CallMethod(obj, "isoformat", "s", "microseconds")); + if (!time_obj) throw py::error_already_set(); + if (!PyUnicode_Check(time_obj.ptr())) { + throw py::type_error("datetime.time.isoformat() must return a str"); + } + Py_ssize_t time_len = PyUnicode_GET_LENGTH(time_obj.ptr()); + info.columnSize = std::max(info.columnSize, time_len); + // PyList_SetItem (lowercase) decrefs the old slot before stealing the new + // reference; safe here because cursor.py already passed a fresh list copy. + if (PyList_SetItem(params, i, time_obj.release().ptr()) != 0) { + throw py::error_already_set(); + } + continue; + } + + // --- Decimal --- + int is_decimal = PyObject_IsInstance(obj, decimal_type); + if (is_decimal == -1) throw py::error_already_set(); + if (is_decimal == 1) { + py::object as_tuple_ptr = steal(PyObject_CallMethod(obj, "as_tuple", NULL)); + if (!as_tuple_ptr) throw py::error_already_set(); + + py::object exponent_obj = steal(PyObject_GetAttrString(as_tuple_ptr.ptr(), "exponent")); + if (!exponent_obj) throw py::error_already_set(); + + // NaN / Infinity / sNaN: refuse rather than silently writing 0. + if (PyUnicode_Check(exponent_obj.ptr())) { + throw py::value_error( + "Cannot bind non-finite Decimal (NaN/Infinity) as SQL NUMERIC"); + } + + py::object digits_obj = steal(PyObject_GetAttrString(as_tuple_ptr.ptr(), "digits")); + if (!digits_obj) throw py::error_already_set(); + + if (!PyTuple_Check(digits_obj.ptr())) { + throw py::type_error("Decimal.as_tuple().digits must be a tuple"); + } + + Py_ssize_t num_digits = PyTuple_GET_SIZE(digits_obj.ptr()); + + // Read the exponent at full width and range-check it BEFORE narrowing to int. + // Decimal exponents are arbitrary-precision, so a value like Decimal("1E+4294967297") + // would otherwise truncate to 1 on LP64, sail past the precision gate below, and + // silently bind 10. An out-of-range exponent cannot produce a bindable NUMERIC at + // any precision, so treat overflow as precision overflow rather than propagating + // OverflowError, matching what the legacy Python path reports. + long long exponent_ll = PyLong_AsLongLong(exponent_obj.ptr()); + if (exponent_ll == -1 && PyErr_Occurred()) { + PyErr_Clear(); + throw py::value_error( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is " + + std::to_string(MAX_NUMERIC_PRECISION) + "."); + } + // Bound before any arithmetic or negation. MAX_NUMERIC_PRECISION on both sides is + // wider than anything bindable, and keeps -exponent well clear of INT_MIN, whose + // negation would be signed-overflow UB. + if (exponent_ll > MAX_NUMERIC_PRECISION || exponent_ll < -MAX_NUMERIC_PRECISION) { + throw py::value_error( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is " + + std::to_string(MAX_NUMERIC_PRECISION) + "."); + } + int exponent = static_cast(exponent_ll); + + // Digit count is likewise capped before it feeds the precision arithmetic. + if (num_digits > MAX_NUMERIC_PRECISION) { + throw py::value_error( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is " + + std::to_string(MAX_NUMERIC_PRECISION) + ", but got " + + std::to_string(num_digits) + "."); + } + + int precision; + // Precision is total base-10 digits after applying Decimal's exponent: positive exponents + // add trailing zeros, in-range negative exponents keep the original digit count, and larger + // negative exponents force leading fractional zeros such as Decimal("0.001") -> precision 3. + if (exponent >= 0) + precision = static_cast(num_digits) + exponent; + else if ((-exponent) <= num_digits) + precision = static_cast(num_digits); + else + precision = -exponent; + + if (precision > MAX_NUMERIC_PRECISION) { + throw py::value_error( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is " + + std::to_string(MAX_NUMERIC_PRECISION) + ", but got " + + std::to_string(precision) + "."); + } + + // Check SMALLMONEY first, then widen to MONEY, so common small values keep the narrowest + // exact range while still accepting larger fixed-point values supported by SQL Server. + // MONEY/SMALLMONEY: SQL Server stores these as fixed-point integers internally. + // We bind as formatted VARCHAR (e.g., "214748.3647") because SQL_C_NUMERIC can't + // represent the exact money range without precision loss on certain ODBC drivers. + // Use exact Decimal comparison (not double) to avoid boundary misclassification. + bool in_money_range = false; + int cmp_ge = PyObject_RichCompareBool(obj, PyTypeCache::smallmoney_min, Py_GE); + int cmp_le = PyObject_RichCompareBool(obj, PyTypeCache::smallmoney_max, Py_LE); + if (cmp_ge == -1 || cmp_le == -1) throw py::error_already_set(); + if (cmp_ge == 1 && cmp_le == 1) { + in_money_range = true; + } else { + cmp_ge = PyObject_RichCompareBool(obj, PyTypeCache::money_min, Py_GE); + cmp_le = PyObject_RichCompareBool(obj, PyTypeCache::money_max, Py_LE); + if (cmp_ge == -1 || cmp_le == -1) throw py::error_already_set(); + if (cmp_ge == 1 && cmp_le == 1) { + in_money_range = true; + } + } + + if (in_money_range) { + py::object formatted = steal(PyObject_CallMethod(obj, "__format__", "s", "f")); + if (!formatted) throw py::error_already_set(); + info.paramSQLType = SQL_VARCHAR; + info.paramCType = PARAM_C_TYPE_TEXT; + info.columnSize = PyUnicode_GET_LENGTH(formatted.ptr()); + info.decimalDigits = 0; + PyObject* raw = formatted.release().ptr(); + if (PyList_SetItem(params, i, raw) != 0) { + // PyList_SetItem steals (decrefs) the item even on failure, + // so raw is already freed — do NOT Py_DECREF here. + throw py::error_already_set(); + } + continue; + } + + // Build SQL_NUMERIC_STRUCT from the Decimal object. Store as a pybind11-castable + // object in the param list so BindParameters can extract it as NumericData. + info.paramSQLType = SQL_NUMERIC; + info.paramCType = SQL_C_NUMERIC; + NumericData nd = build_numeric_data(as_tuple_ptr.ptr(), digits_obj.ptr(), exponent); + info.columnSize = nd.precision; + info.decimalDigits = nd.scale; + // Store NumericData as a Python object in the param list for the binder. + py::object numeric_obj = py::cast(nd); + PyObject* raw = numeric_obj.release().ptr(); + if (PyList_SetItem(params, i, raw) != 0) { + // PyList_SetItem steals (decrefs) the item even on failure. + throw py::error_already_set(); + } + continue; + } + + // --- UUID --- + int is_uuid = PyObject_IsInstance(obj, uuid_type); + if (is_uuid == -1) throw py::error_already_set(); + if (is_uuid == 1) { + PyObject* bytes_le = PyObject_GetAttrString(obj, "bytes_le"); + if (!bytes_le) throw py::error_already_set(); + info.paramSQLType = SQL_GUID; + info.paramCType = SQL_C_GUID; + info.columnSize = 16; + info.decimalDigits = 0; + if (PyList_SetItem(params, i, bytes_le) != 0) { + // PyList_SetItem steals (decrefs) the item even on failure. + throw py::error_already_set(); + } + continue; + } + + // --- Unknown type: raise TypeError (matches Python _map_sql_type) --- + throw py::type_error( + "Unsupported parameter type: The driver cannot safely convert it to a SQL type."); + } + + return infos; +} + +// Helper: build SQL_NUMERIC_STRUCT from an already-unpacked Decimal.as_tuple(). +// +// Callers in DetectParamTypes have already called as_tuple() and pulled out the digits +// tuple and exponent, so those are passed in rather than re-entering Python to fetch +// them a second time. +// +// The mantissa is accumulated into a fixed 128-bit value held as four 32-bit limbs +// instead of Python bigint arithmetic. SQL Server caps NUMERIC precision at +// MAX_NUMERIC_PRECISION (38) digits and callers reject anything larger, so the value +// always fits the 16 bytes SQL_NUMERIC_STRUCT provides. Limbs keep this portable +// (MSVC has no __int128) and the result is written out byte-by-byte so host endianness +// does not matter. +inline NumericData build_numeric_data(PyObject* as_tuple, PyObject* digits, int exponent) { + py::object sign_obj = steal(PyObject_GetAttrString(as_tuple, "sign")); + if (!sign_obj) throw py::error_already_set(); + int sign_val = static_cast(PyLong_AsLong(sign_obj.ptr())); + if (sign_val == -1 && PyErr_Occurred()) throw py::error_already_set(); + + if (!PyTuple_Check(digits)) { + throw py::type_error("Decimal.as_tuple().digits must be a tuple"); + } + + // SQL Server precision counts all stored decimal digits, while scale is just the + // fractional digits. A positive exponent moves trailing zeros into the integer part; + // a negative exponent means scale = -exponent and precision must still cover leading + // fractional zeros such as 0.001. + const Py_ssize_t digit_count = PyTuple_GET_SIZE(digits); + const int num_digits = static_cast(digit_count); + int precision, scale; + if (exponent >= 0) { + precision = num_digits + exponent; + scale = 0; + } else { + scale = -exponent; + precision = std::max(num_digits, scale); + } + precision = std::max(1, std::min(precision, MAX_NUMERIC_PRECISION)); + scale = std::min(scale, precision); + + // 128-bit magnitude as four little-endian 32-bit limbs. Returns the carry out of the + // top limb, which is non-zero only if the value overflowed 128 bits. + uint32_t limb[4] = {0, 0, 0, 0}; + auto mul10_add = [&limb](uint32_t addend) -> uint64_t { + uint64_t carry = addend; + for (int k = 0; k < 4; ++k) { + uint64_t cur = static_cast(limb[k]) * 10u + carry; + limb[k] = static_cast(cur); + carry = cur >> 32; + } + return carry; + }; + + uint64_t overflow = 0; + for (Py_ssize_t i = 0; i < digit_count; ++i) { + PyObject* digit_obj = PyTuple_GET_ITEM(digits, i); + long digit = PyLong_AsLong(digit_obj); + if (digit == -1 && PyErr_Occurred()) throw py::error_already_set(); + overflow |= mul10_add(static_cast(digit)); + } + // A positive exponent means as_tuple() omitted trailing zeros, so Decimal("123e2") + // must become mantissa 12300 before packing. + for (int j = 0; j < exponent; ++j) { + overflow |= mul10_add(0); + } + if (overflow != 0) { + throw py::value_error("Decimal magnitude exceeds the 16-byte SQL NUMERIC capacity"); + } + + NumericData nd; + nd.precision = static_cast(precision); + nd.scale = static_cast(scale); + // SQL uses sign=1 for positive and sign=0 for negative, the inverse of + // Decimal.as_tuple().sign. + nd.sign = (sign_val == 0) ? 1 : 0; + nd.val.assign(SQL_MAX_NUMERIC_LEN, '\0'); + for (int k = 0; k < 4; ++k) { + nd.val[k * 4 + 0] = static_cast(limb[k] & 0xFF); + nd.val[k * 4 + 1] = static_cast((limb[k] >> 8) & 0xFF); + nd.val[k * 4 + 2] = static_cast((limb[k] >> 16) & 0xFF); + nd.val[k * 4 + 3] = static_cast((limb[k] >> 24) & 0xFF); + } + return nd; +} diff --git a/mssql_python/pybind/py_ref.hpp b/mssql_python/pybind/py_ref.hpp new file mode 100644 index 00000000..f4fce618 --- /dev/null +++ b/mssql_python/pybind/py_ref.hpp @@ -0,0 +1,32 @@ +// py_ref.hpp — Adopting raw CPython references into pybind11's RAII ownership. +// +// The CPython C API hands back two kinds of PyObject*: NEW references, which the +// caller owns and must release, and BORROWED references, which the caller must +// incref before holding. Getting that wrong is a leak in one direction and a +// use-after-free in the other. These two helpers make the choice explicit at +// every call site and hand ownership to py::object, whose destructor then does +// the releasing. +// +// Named after nanobind's nb::steal / nb::borrow, which have the same signature, +// so a future migration is a namespace change rather than a rewrite. + +#pragma once +#include +#include + +namespace py = pybind11; + +// Takes ownership of a NEW reference without increfing. Correct for the common +// returns-a-new-reference calls: PyObject_GetAttrString, PyObject_CallMethod, +// PyImport_ImportModule, PyUnicode_FromString. +// +// Applying it to a BORROWED reference (PyList_GetItem, PyTuple_GetItem, +// PyDict_GetItem, PyDict_GetItemString) is a premature decref and a +// use-after-free. Use borrow() for those. +template +inline T steal(PyObject* p) { return py::reinterpret_steal(py::handle(p)); } + +// Increfs a BORROWED reference so it can be held safely. Safe on a new reference +// only if the caller still decrefs the original, which it usually should not. +template +inline T borrow(PyObject* p) { return py::reinterpret_borrow(py::handle(p)); } diff --git a/mssql_python/pybind/py_type_cache.hpp b/mssql_python/pybind/py_type_cache.hpp new file mode 100644 index 00000000..19a9c1f0 --- /dev/null +++ b/mssql_python/pybind/py_type_cache.hpp @@ -0,0 +1,113 @@ +// py_type_cache.hpp — One-time cache of Python type objects and MONEY boundary constants. +// +// Called on first execute(). Uses raw CPython API (not pybind11) because +// these cached PyObject* are compared via PyObject_IsInstance in the +// hot DetectParamTypes loop — wrapping them in py::object would add +// unnecessary ref-count traffic on every parameter. +// +// All cached pointers are module-lifetime singletons (never DECREFed). + +#pragma once +#include +#include +#include + +#include "py_ref.hpp" // steal / borrow + +namespace py = pybind11; + +namespace PyTypeCache { + +// Module-lifetime singletons — never DECREFed, alive for the process. +inline PyObject* datetime_class = nullptr; +inline PyObject* date_class = nullptr; +inline PyObject* time_class = nullptr; +inline PyObject* decimal_class = nullptr; +inline PyObject* uuid_class = nullptr; +inline PyObject* money_min = nullptr; +inline PyObject* money_max = nullptr; +inline PyObject* smallmoney_min = nullptr; +inline PyObject* smallmoney_max = nullptr; +inline bool cache_initialized = false; + +// Import a module and extract an attribute. Returns a new reference. +inline PyObject* import_attr(const char* module_name, const char* attr_name) { + py::object mod = steal(PyImport_ImportModule(module_name)); + if (!mod) throw py::error_already_set(); + PyObject* attr = PyObject_GetAttrString(mod.ptr(), attr_name); + if (!attr) throw py::error_already_set(); + return attr; +} + +// Return cached type, falling back to a fresh import for callers that run before +// initialize() has. The fallback exists for the legacy execute path, which does its +// type detection in Python and can therefore reach here without the cache being warm; +// it can be dropped once that path is removed. +inline PyObject* get_cached_class(PyObject* cached, const char* module_name, const char* attr_name) { + if (cache_initialized && cached) return cached; + py::object mod = steal(PyImport_ImportModule(module_name)); + if (!mod) return nullptr; + return PyObject_GetAttrString(mod.ptr(), attr_name); +} + +// One-time init. Uses local py::objects so exception cleanup is automatic; +// only .release() into globals after ALL acquisitions succeed. +inline void initialize() { + if (cache_initialized) return; + + PyDateTime_IMPORT; + if (PyDateTimeAPI == nullptr) throw py::error_already_set(); + + py::object dt_mod = steal(PyImport_ImportModule("datetime")); + if (!dt_mod) throw py::error_already_set(); + + py::object dt_cls = steal(PyObject_GetAttrString(dt_mod.ptr(), "datetime")); + py::object date_cls = steal(PyObject_GetAttrString(dt_mod.ptr(), "date")); + py::object time_cls = steal(PyObject_GetAttrString(dt_mod.ptr(), "time")); + if (!dt_cls || !date_cls || !time_cls) throw py::error_already_set(); + + py::object dec_cls = steal(import_attr("decimal", "Decimal")); + py::object uuid_cls = steal(import_attr("uuid", "UUID")); + + // Pre-compute MONEY/SMALLMONEY boundary Decimals for exact comparison + // in DetectParamTypes (avoids double-precision boundary errors). + py::object sm_min = steal(PyObject_CallFunction(dec_cls.ptr(), "s", "-214748.3648")); + py::object sm_max = steal(PyObject_CallFunction(dec_cls.ptr(), "s", "214748.3647")); + py::object m_min = steal(PyObject_CallFunction(dec_cls.ptr(), "s", "-922337203685477.5808")); + py::object m_max = steal(PyObject_CallFunction(dec_cls.ptr(), "s", "922337203685477.5807")); + if (!sm_min || !sm_max || !m_min || !m_max) throw py::error_already_set(); + + // Commit to globals — all acquisitions succeeded. + datetime_class = dt_cls.release().ptr(); + date_class = date_cls.release().ptr(); + time_class = time_cls.release().ptr(); + decimal_class = dec_cls.release().ptr(); + uuid_class = uuid_cls.release().ptr(); + smallmoney_min = sm_min.release().ptr(); + smallmoney_max = sm_max.release().ptr(); + money_min = m_min.release().ptr(); + money_max = m_max.release().ptr(); + cache_initialized = true; +} + +// Wrap a cached pointer as py::object. A cached class is a module-lifetime +// singleton we do not own, so borrow it; the fallback import path returns a new +// reference, so steal it. +inline py::object wrap_cached_or_imported(PyObject* obj) { + if (!obj) throw py::error_already_set(); + return cache_initialized ? borrow(obj) : steal(obj); +} + +inline PyObject* get_datetime_class() { return get_cached_class(datetime_class, "datetime", "datetime"); } +inline PyObject* get_date_class() { return get_cached_class(date_class, "datetime", "date"); } +inline PyObject* get_time_class() { return get_cached_class(time_class, "datetime", "time"); } +inline PyObject* get_decimal_class() { return get_cached_class(decimal_class, "decimal", "Decimal"); } +inline PyObject* get_uuid_class() { return get_cached_class(uuid_class, "uuid", "UUID"); } + +inline py::object get_datetime_class_obj() { return wrap_cached_or_imported(get_datetime_class()); } +inline py::object get_date_class_obj() { return wrap_cached_or_imported(get_date_class()); } +inline py::object get_time_class_obj() { return wrap_cached_or_imported(get_time_class()); } +inline py::object get_decimal_class_obj() { return wrap_cached_or_imported(get_decimal_class()); } +inline py::object get_uuid_class_obj() { return wrap_cached_or_imported(get_uuid_class()); } + +} // namespace PyTypeCache diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index f756e89b..0630a35b 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -14381,32 +14381,22 @@ def test_xml_malformed_input(cursor, db_connection): def test_decimal_special_values_coverage(cursor): - """Test decimal processing with special values like NaN and Infinity (Lines 213-221).""" + """Non-finite Decimals are rejected explicitly by `_get_numeric_data`.""" from decimal import Decimal - # Test special decimal values that have string exponents + # NaN reports exponent 'n', sNaN reports 'N', Infinity reports 'F'. None of + # them has a SQL NUMERIC encoding, so all three must raise ValueError rather + # than falling through to precision=38 and packing a silent zero. test_values = [ - Decimal("NaN"), # Should have str exponent 'n' - Decimal("Infinity"), # Should have str exponent 'F' - Decimal("-Infinity"), # Should have str exponent 'F' + Decimal("NaN"), + Decimal("sNaN"), + Decimal("Infinity"), + Decimal("-Infinity"), ] for special_val in test_values: - try: - # This should trigger the special value handling path (lines 217-218) - # But there's a bug in the code - it doesn't handle string exponents properly after line 218 + with pytest.raises(ValueError, match="non-finite"): cursor._get_numeric_data(special_val) - except (ValueError, TypeError) as e: - # Expected - either ValueError for unsupported values or TypeError due to str/int comparison - # This exercises the special value code path (lines 217-218) even though it errors later - assert ( - "not supported" in str(e) - or "Precision of the numeric value is too high" in str(e) - or "'>' not supported between instances of 'str' and 'int'" in str(e) - ) - except Exception as e: - # Other exceptions are also acceptable as we're testing error paths - pass def test_decimal_negative_exponent_edge_cases(cursor): diff --git a/tests/test_010_pybind_functions.py b/tests/test_010_pybind_functions.py index 106b64ca..b7374112 100644 --- a/tests/test_010_pybind_functions.py +++ b/tests/test_010_pybind_functions.py @@ -513,6 +513,7 @@ def test_all_exposed_functions_exist(self): "DDBCSetDecimalSeparator", "DDBCSQLExecDirect", "DDBCSQLExecute", + "DDBCSQLExecuteLegacy", "DDBCSQLRowCount", "DDBCSQLFetch", "DDBCSQLNumResultCols", diff --git a/tests/test_023_execute_path_parity.py b/tests/test_023_execute_path_parity.py new file mode 100644 index 00000000..ac3bfe56 --- /dev/null +++ b/tests/test_023_execute_path_parity.py @@ -0,0 +1,774 @@ +""" +Coverage for the two parameter paths, each tested on its own terms — no path is +forced through a door real callers do not use. + +1. Native path (C++ DetectParamTypes + DDBCSQLExecute) — the default that ~99% of + calls take. Exercised end to end through plain ``cursor.execute(...)``. +2. Python type detection (``_map_sql_type`` / ``_get_numeric_data``) — the reference + the native path was ported from. Asserted directly as a pure function: value in, + (SQL type, C type, column size, decimal digits, DAE) out. No DB round-trip, so + the assertion cannot be masked by SQL Server coercing a wrong-but-convertible + type back to the right value. +3. Legacy execute path (DDBCSQLExecuteLegacy) — only reachable by a caller through + ``setinputsizes()``, so it is tested through exactly that API, using it for what + it is for: honouring user-supplied type overrides. + +Uses the project's `cursor` fixture from conftest.py so the tests work in any +environment that runs the rest of the suite. +""" + +import datetime +import decimal +import gc +import uuid +import weakref + +import pytest + +from mssql_python.constants import ConstantsDDBC as ddbc_sql_const + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _standard_roundtrip(cursor, value): + """Native path: no setinputsizes, so C++ detects the types.""" + cursor.execute("SELECT ?", [value]) + return cursor.fetchone()[0] + + +def _override_roundtrip(cursor, value, sql_type, column_size): + """Legacy execute path through its real entry point. + + ``setinputsizes`` is the only public API that routes a call to + DDBCSQLExecuteLegacy. It also declares the parameter's type explicitly, so this + helper tests the user-override contract — not type *detection*, which is covered + directly against ``_map_sql_type`` elsewhere in this file.""" + cursor.setinputsizes([(sql_type, column_size, 0)]) + try: + cursor.execute("SELECT ?", [value]) + return cursor.fetchone()[0] + finally: + cursor.setinputsizes(None) + + +# --------------------------------------------------------------------------- +# Standard-path coverage: representative type matrix +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + # int range detection (TINYINT / SMALLINT / INTEGER / BIGINT) + 0, + 1, + 255, + 256, + 32767, + 32768, + 2147483647, + 2147483648, + -1, + -32768, + -2147483648, + # bool + True, + False, + # float + 0.0, + 3.14, + -1.5e10, + # str (ASCII inline) + "", + "hello", + "a" * 100, + # bytes + b"", + b"\x00\x01\x02", + b"x" * 100, + ], +) +def test_standard_path_basic_types(cursor, value): + """Standard path round-trips representative scalar types correctly.""" + result = _standard_roundtrip(cursor, value) + assert result == value, ( + f"Standard-path roundtrip mismatch for {type(value).__name__} {value!r}: " f"got {result!r}" + ) + + +# --------------------------------------------------------------------------- +# Subclass support — regression for the *_CheckExact bug from PR review +# --------------------------------------------------------------------------- + + +def test_int_subclass(cursor): + class MyInt(int): + pass + + assert _standard_roundtrip(cursor, MyInt(42)) == 42 + + +def test_str_subclass(cursor): + class MyStr(str): + pass + + assert _standard_roundtrip(cursor, MyStr("hello")) == "hello" + + +def test_bytes_subclass(cursor): + class MyBytes(bytes): + pass + + assert _standard_roundtrip(cursor, MyBytes(b"hello")) == b"hello" + + +def test_float_subclass(cursor): + class MyFloat(float): + pass + + assert _standard_roundtrip(cursor, MyFloat(3.14)) == 3.14 + + +# --------------------------------------------------------------------------- +# Caller-list isolation and refcount safety +# --------------------------------------------------------------------------- + + +def test_caller_param_list_not_mutated(cursor): + """DetectParamTypes must not mutate the caller's parameter list.""" + params = ["hello", 42, 3.14, datetime.date(2024, 1, 1), uuid.uuid4()] + snapshot = list(params) + cursor.execute("SELECT ?, ?, ?, ?, ?", params) + cursor.fetchone() + assert params == snapshot, f"Caller list was mutated: {params} != {snapshot}" + + +def test_no_refcount_leak_on_in_place_replacement(cursor): + """Decimal/UUID/time params get replaced in-place inside DetectParamTypes + via PyList_SetItem. The replaced object must have its reference dropped — + a regression caught in PR review where PyList_SET_ITEM (uppercase, no + decref) leaked one reference per replaced item per execute.""" + + class TrackedDec(decimal.Decimal): + pass + + td = TrackedDec("123.45") + ref = weakref.ref(td) + params = [td] + del td # drop our local strong reference + + cursor.execute("SELECT ?", params) + cursor.fetchone() + del params # drop the list's strong reference + gc.collect() + + assert ref() is None, ( + "Decimal parameter was leaked: PyList_SetItem must decref the old " + "slot before stealing the new reference." + ) + + +# --------------------------------------------------------------------------- +# Error semantics +# --------------------------------------------------------------------------- + + +def test_unsupported_type_raises_typeerror(cursor): + """Standard path must raise TypeError for unknown parameter types — matching + the legacy path's `_map_sql_type` final branch.""" + with pytest.raises(TypeError): + cursor.execute("SELECT ?", [{1, 2, 3}]) # set is not bindable + + +def test_decimal_nan_rejected(cursor): + """Non-finite Decimals must raise rather than silently bind as 0.""" + with pytest.raises(ValueError): + cursor.execute("SELECT ?", [decimal.Decimal("NaN")]) + + +@pytest.mark.parametrize( + "exp", + [ + 2**32 + 1, # truncated to 1 by a 32-bit narrowing cast + 2**31, # truncates to exactly INT_MIN; negating that is signed-overflow UB + 2**31 - 1, # INT_MAX + -(2**32 + 1), + -(2**31), + 39, # first out-of-range exponent that needs no truncation to be invalid + -39, + ], +) +def test_decimal_out_of_range_exponent_rejected(cursor, exp): + """Exponents beyond SQL Server's 38-digit precision must raise, including ones + that only look valid after a 32-bit narrowing cast. + + Regression guard: the exponent used to be cast to int before being range + checked, so Decimal("1E+4294967297") truncated to 1, passed the precision + gate, and silently bound 10 while the legacy path raised. + """ + with pytest.raises(Exception) as excinfo: + cursor.execute("SELECT ?", [decimal.Decimal(f"1E{exp:+d}")]) + cursor.fetchone() + # the failure must be about precision, not an OverflowError leaking from the cast + assert not isinstance(excinfo.value, OverflowError) + + +@pytest.mark.parametrize("exp", [37, -38, 0]) +def test_decimal_in_range_exponent_still_binds(cursor, exp): + """The range check must not reject exponents SQL Server can represent.""" + value = decimal.Decimal(f"1E{exp:+d}") + cursor.execute("SELECT ?", [value]) + assert cursor.fetchone()[0] is not None + + +def test_aware_time_matches_legacy(cursor): + """A tz-aware datetime.time must behave the same on both paths. + + SQL Server's TIME has no UTC offset, so isoformat's "+05:30" cannot bind and + both paths reject it. The standard path used to hand-format the raw H/M/S/us + fields, which silently dropped the offset and bound a different time than the + caller passed while the legacy path raised. + """ + aware = datetime.time( + 1, 2, 3, 4, tzinfo=datetime.timezone(datetime.timedelta(hours=5, minutes=30)) + ) + with pytest.raises(Exception): + cursor.execute("SELECT ?", [aware]) + cursor.fetchone() + + +def test_naive_time_roundtrips(cursor): + """Naive times are unaffected by the aware-time handling above.""" + naive = datetime.time(1, 2, 3, 4) + assert _standard_roundtrip(cursor, naive) == naive + + +# --------------------------------------------------------------------------- +# Standard-vs-legacy parity for representative types +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Legacy execute path: user-supplied type overrides via setinputsizes +# +# These are the only tests that use setinputsizes, and they use it for its real +# purpose. They keep DDBCSQLExecuteLegacy and the explicit-override branch of +# _create_parameter_types_list covered end to end. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value, sql_type, column_size", + [ + ("hello", ddbc_sql_const.SQL_VARCHAR.value, 5), + (42, ddbc_sql_const.SQL_INTEGER.value, 0), + (3.14, ddbc_sql_const.SQL_DOUBLE.value, 0), + (b"data", ddbc_sql_const.SQL_VARBINARY.value, 4), + ], +) +def test_setinputsizes_override_roundtrips(cursor, value, sql_type, column_size): + """A user-declared type via setinputsizes round-trips through the legacy path.""" + assert _override_roundtrip(cursor, value, sql_type, column_size) == value + + +def test_setinputsizes_shorter_than_params_detects_the_rest(cursor): + """setinputsizes with fewer entries than parameters: covered indices use the + override, uncovered ones fall through to _map_sql_type inside + _create_parameter_types_list. This is the only end-to-end route to that + detection fallback on the legacy path, so it keeps that line covered. + + (A None entry cannot be used here — setinputsizes validates and rejects None.) + """ + cursor.setinputsizes([(ddbc_sql_const.SQL_VARCHAR.value, 5, 0)]) + try: + with pytest.warns(Warning): # count mismatch is warned, then execution proceeds + cursor.execute("SELECT ?, ?", ["hello", 42]) + row = cursor.fetchone() + assert row[0] == "hello" + assert row[1] == 42 + finally: + cursor.setinputsizes(None) + + +# --------------------------------------------------------------------------- +# Edge case tests (issues caught in rubber-duck review) +# --------------------------------------------------------------------------- + + +def test_large_bytearray_dae(cursor): + """Large bytearray (>8000 bytes) must stream via DAE without crashing. + This catches the pybind11 bytes-cast-from-bytearray bug.""" + large_ba = bytearray(b"\xab" * 10000) + cursor.execute("SELECT DATALENGTH(CAST(? AS VARBINARY(MAX)))", [large_ba]) + result = cursor.fetchone()[0] + assert result == 10000 + + +def test_large_bytes_dae(cursor): + """Large bytes (>8000 bytes) must stream via DAE correctly.""" + large_b = b"\xcd" * 10000 + cursor.execute("SELECT DATALENGTH(CAST(? AS VARBINARY(MAX)))", [large_b]) + result = cursor.fetchone()[0] + assert result == 10000 + + +def test_large_string_dae(cursor): + """Large string (>4000 chars) must stream via DAE correctly.""" + large_str = "x" * 5000 + cursor.execute("SELECT LEN(?)", [large_str]) + result = cursor.fetchone()[0] + assert result == 5000 + + +def test_large_unicode_string_dae(cursor): + """Large unicode string (>4000 UTF-16 code units) streams via DAE.""" + large_str = "\u00e9" * 5000 # é = 1 UTF-16 code unit each + cursor.execute("SELECT LEN(?)", [large_str]) + result = cursor.fetchone()[0] + assert result == 5000 + + +@pytest.mark.parametrize( + "value", + [ + decimal.Decimal("-922337203685477.5808"), # MONEY_MIN boundary + decimal.Decimal("922337203685477.5807"), # MONEY_MAX boundary + decimal.Decimal("-214748.3648"), # SMALLMONEY_MIN + decimal.Decimal("214748.3647"), # SMALLMONEY_MAX + decimal.Decimal("0.01"), # typical money value + ], +) +def test_decimal_money_boundary(cursor, value): + """Decimal values at MONEY/SMALLMONEY boundaries must round-trip correctly.""" + cursor.execute("SELECT CAST(? AS DECIMAL(38,4))", [value]) + result = cursor.fetchone()[0] + assert result == value, f"MONEY boundary mismatch: sent {value}, got {result}" + + +def test_decimal_outside_money_uses_numeric(cursor): + """Decimal outside MONEY range must use SQL_NUMERIC binding.""" + # One unit above MONEY_MAX + value = decimal.Decimal("922337203685477.5808") + cursor.execute("SELECT CAST(? AS DECIMAL(38,4))", [value]) + result = cursor.fetchone()[0] + assert result == value + + +def test_decimal_infinity_rejected(cursor): + """Decimal('Infinity') must raise, not silently bind as 0.""" + with pytest.raises(ValueError): + cursor.execute("SELECT ?", [decimal.Decimal("Infinity")]) + + +def test_binary_with_embedded_nulls(cursor): + """Binary data with embedded null bytes must not be truncated.""" + data = b"\x00\x01\x00\x02\x00" + cursor.execute("SELECT DATALENGTH(CAST(? AS VARBINARY(MAX)))", [data]) + result = cursor.fetchone()[0] + assert result == 5 + + +def test_string_with_embedded_nulls(cursor): + """String with embedded NUL chars must not be truncated.""" + value = "hello\x00world" + cursor.execute("SELECT LEN(?)", [value]) + result = cursor.fetchone()[0] + assert result == 11 + + +def test_integer_overflow_detected(cursor): + """Integers beyond int64 range trigger overflow detection in C++ and bind as BIGINT. + SQL Server cannot store values outside [-2^63, 2^63-1] in BIGINT, so these raise.""" + with pytest.raises(Exception): + cursor.execute("SELECT ?", [2**63]) + with pytest.raises(Exception): + cursor.execute("SELECT ?", [-(2**63) - 1]) + + +@pytest.mark.parametrize("value", [decimal.Decimal("NaN"), decimal.Decimal("sNaN")]) +def test_decimal_nan_variants_rejected(cursor, value): + """Decimal NaN variants must raise, not silently bind as 0.""" + with pytest.raises(ValueError): + cursor.execute("SELECT ?", [value]) + + +NON_FINITE_DECIMALS = [ + decimal.Decimal("NaN"), + decimal.Decimal("-NaN"), + decimal.Decimal("sNaN"), + decimal.Decimal("Infinity"), + decimal.Decimal("-Infinity"), +] + + +@pytest.mark.parametrize("value", NON_FINITE_DECIMALS, ids=str) +def test_non_finite_decimal_exception_type_parity(cursor, value): + """Both paths must reject non-finite Decimals with the *same* exception type. + + Before this was made explicit, the two paths diverged on type: the native + detector raised ValueError, while the legacy path raised decimal.InvalidOperation + for NaN (from the MONEY range comparison) and TypeError for Infinity (from + comparing a str exponent against an int inside `_get_numeric_data`). Callers + writing `except ValueError` therefore saw different behaviour depending on + whether setinputsizes happened to be set. + """ + # Native path — C++ DetectParamTypes, reached through execute(). + with pytest.raises(ValueError) as native_exc: + cursor.execute("SELECT ?", [value]) + + # Legacy path — Python type detection. Called directly because setinputsizes, + # the only way to reach the legacy branch from execute(), bypasses _map_sql_type. + params = [value] + with pytest.raises(ValueError) as legacy_exc: + cursor._map_sql_type(value, params, 0) + + assert "non-finite" in str(native_exc.value).lower() + assert "non-finite" in str(legacy_exc.value).lower() + + +@pytest.mark.parametrize("value", NON_FINITE_DECIMALS, ids=str) +def test_get_numeric_data_rejects_non_finite(cursor, value): + """`_get_numeric_data` is also reachable from executemany's typing pass, so it + needs the same explicit rejection rather than falling through to precision=38 + and packing a silent zero.""" + with pytest.raises(ValueError, match="non-finite"): + cursor._get_numeric_data(value) + + +def test_decimal_precision_overflow_rejected(cursor): + """Decimals beyond SQL Server's max precision must raise.""" + with pytest.raises(Exception): + cursor.execute("SELECT ?", [decimal.Decimal("123456789012345678901234567890123456789")]) + + +# --------------------------------------------------------------------------- +# Text parameter C-type parity (all platforms bind wide) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", + [ + "", + "plain ascii", + "a" * 4000, # inline boundary + "a" * 4001, # DAE boundary + "café", # non-ASCII, forces the unicode branch + "naïve café ☕", + "日本語テキスト", + "mixed ascii and 日本語", + "with 'quote' and \"double\"", + ], + ids=repr, +) +def test_text_params_bind_wide_on_every_platform(cursor, value): + """Text parameters must survive a round-trip identically on every platform. + + The native detector used to resolve its text C type to a real `SQL_C_CHAR (1)` + on Windows while using `SQL_C_WCHAR` on Linux/macOS. The legacy path binds with + the Python layer's `SQL_C_CHAR` constant, which is numerically -8 — that is + ODBC's `SQL_C_WCHAR` — so the legacy path has always bound wide everywhere. + Windows was therefore the only platform where the two paths disagreed on C type + and on the driver-side encoding path. This test pins the round-trip behaviour so + a reintroduced narrow binding shows up as a Windows-only failure. + """ + cursor.execute("SELECT ?", [value]) + assert cursor.fetchone()[0] == value + + +def test_ascii_text_roundtrip_into_nvarchar_column(cursor): + """ASCII strings take the `SQL_VARCHAR` + wide-C-type combination, which is the + exact pairing that differed on Windows. Round-trip through a real NVARCHAR column + so the driver's conversion is exercised, not just SELECT ? echo.""" + cursor.execute("SELECT CAST(? AS NVARCHAR(100))", ["ascii only"]) + assert cursor.fetchone()[0] == "ascii only" + + cursor.execute("SELECT CAST(? AS NVARCHAR(100))", ["café ☕"]) + assert cursor.fetchone()[0] == "café ☕" + + +def test_time_param_binds_wide(cursor): + """`datetime.time` is normalized to a string and bound with the same text C type, + so it shares the Windows narrow/wide divergence.""" + value = datetime.time(1, 2, 3, 4) + cursor.execute("SELECT CAST(? AS TIME(6))", [value]) + assert cursor.fetchone()[0] == value + + +def test_money_range_decimal_binds_wide(cursor): + """Decimals inside the MONEY range are formatted to text and bound with the text + C type, the third consumer of the platform-dependent constant.""" + value = decimal.Decimal("214748.3647") + cursor.execute("SELECT CAST(? AS MONEY)", [value]) + assert cursor.fetchone()[0] == value + + +# --------------------------------------------------------------------------- +# Python type detection, asserted directly as a pure function. +# +# _map_sql_type(value, [value], 0) returns the 5-tuple +# (SQL type, C type, column size, decimal digits, DAE) +# with no DB round-trip, so a wrong type cannot be hidden by SQL Server coercing +# the value back. A fresh single-element list is passed per call because the +# function mutates its slot in place for numeric / uuid / money / time. +# --------------------------------------------------------------------------- + +_c = ddbc_sql_const + + +def _detect(cursor, value): + return cursor._map_sql_type(value, [value], 0) + + +def _param_basetype(cursor, value): + """The SQL Server base type a parameter arrives as, observed from outside. + + CAST(? AS sql_variant) preserves the parameter's declared SQL type, so this + returns 'varchar' vs 'nvarchar' for the native path without needing a test-only + detector. Limited to values that fit in sql_variant's 8000-byte cap, so it works + for the small/inline cases only.""" + cursor.execute("SELECT SQL_VARIANT_PROPERTY(CAST(? AS sql_variant), 'BaseType')", [value]) + return cursor.fetchone()[0] + + +# Deterministic cases: the full 5-tuple is fixed by the type alone. +DETECTION_CASES = [ + # None and bool + (None, _c.SQL_UNKNOWN_TYPE, _c.SQL_C_DEFAULT, 1, 0, False), + (True, _c.SQL_BIT, _c.SQL_C_BIT, 1, 0, False), + (False, _c.SQL_BIT, _c.SQL_C_BIT, 1, 0, False), + # int width detection + (0, _c.SQL_TINYINT, _c.SQL_C_TINYINT, 3, 0, False), + (255, _c.SQL_TINYINT, _c.SQL_C_TINYINT, 3, 0, False), + (256, _c.SQL_SMALLINT, _c.SQL_C_SHORT, 5, 0, False), + (-1, _c.SQL_SMALLINT, _c.SQL_C_SHORT, 5, 0, False), + (32767, _c.SQL_SMALLINT, _c.SQL_C_SHORT, 5, 0, False), + (32768, _c.SQL_INTEGER, _c.SQL_C_LONG, 10, 0, False), + (-32769, _c.SQL_INTEGER, _c.SQL_C_LONG, 10, 0, False), + (2147483647, _c.SQL_INTEGER, _c.SQL_C_LONG, 10, 0, False), + (2147483648, _c.SQL_BIGINT, _c.SQL_C_SBIGINT, 19, 0, False), + (-2147483649, _c.SQL_BIGINT, _c.SQL_C_SBIGINT, 19, 0, False), + # float + (3.14, _c.SQL_DOUBLE, _c.SQL_C_DOUBLE, 15, 0, False), + # small binary + (b"", _c.SQL_VARBINARY, _c.SQL_C_BINARY, 1, 0, False), + (b"abc", _c.SQL_VARBINARY, _c.SQL_C_BINARY, 3, 0, False), + # date / datetime / time + (datetime.date(2024, 1, 1), _c.SQL_DATE, _c.SQL_C_TYPE_DATE, 10, 0, False), + ( + datetime.datetime(2024, 1, 1, 2, 3, 4), + _c.SQL_TIMESTAMP, + _c.SQL_C_TYPE_TIMESTAMP, + 26, + 6, + False, + ), + (datetime.time(1, 2, 3), _c.SQL_TYPE_TIME, _c.SQL_C_CHAR, 16, 6, False), +] + + +@pytest.mark.parametrize( + "value, sql_type, c_type, column_size, decimal_digits, is_dae", + DETECTION_CASES, + ids=[repr(row[0]) for row in DETECTION_CASES], +) +def test_map_sql_type_detection( + cursor, value, sql_type, c_type, column_size, decimal_digits, is_dae +): + assert _detect(cursor, value) == ( + sql_type.value, + c_type.value, + column_size, + decimal_digits, + is_dae, + ) + + +def test_map_sql_type_uuid(cursor): + """UUID → SQL_GUID, and the slot is replaced with its little-endian bytes.""" + u = uuid.uuid4() + params = [u] + assert cursor._map_sql_type(u, params, 0) == ( + _c.SQL_GUID.value, + _c.SQL_C_GUID.value, + 16, + 0, + False, + ) + assert params[0] == u.bytes_le + + +@pytest.mark.parametrize( + "value, sql_type, c_type, column_size, is_dae", + [ + ("", _c.SQL_VARCHAR, _c.SQL_C_CHAR, 0, False), + ("hello", _c.SQL_VARCHAR, _c.SQL_C_CHAR, 5, False), + ("a" * 4000, _c.SQL_VARCHAR, _c.SQL_C_CHAR, 4000, False), # inline boundary + ("a" * 4001, _c.SQL_VARCHAR, _c.SQL_C_CHAR, 0, True), # ASCII DAE + ("café", _c.SQL_WVARCHAR, _c.SQL_C_WCHAR, 4, False), # unicode inline + ("é" * 4001, _c.SQL_WVARCHAR, _c.SQL_C_WCHAR, 0, True), # unicode DAE + ], + ids=["empty", "ascii", "ascii-4000", "ascii-4001-dae", "unicode", "unicode-dae"], +) +def test_map_sql_type_strings(cursor, value, sql_type, c_type, column_size, is_dae): + assert _detect(cursor, value) == (sql_type.value, c_type.value, column_size, 0, is_dae) + + +@pytest.mark.parametrize( + "prefix", ["POINT(1 2)", "LINESTRING(0 0, 1 1)", "POLYGON((0 0,1 0,1 1,0 0))"] +) +def test_map_sql_type_geometry_wkt(cursor, prefix): + """Legacy detection: geometry WKT is SQL_WVARCHAR regardless of the unicode heuristic.""" + assert _detect(cursor, prefix) == ( + _c.SQL_WVARCHAR.value, + _c.SQL_C_WCHAR.value, + len(prefix), + 0, + False, + ) + + +@pytest.mark.parametrize( + "prefix", ["POINT(1 2)", "LINESTRING(0 0, 1 1)", "POLYGON((0 0,1 0,1 1,0 0))"] +) +def test_native_small_geometry_binds_nvarchar(cursor, prefix): + """Native path: small geometry WKT arrives as nvarchar, while a plain ASCII string + arrives as varchar. This is the observable proxy that native geometry detection + fires and picks the wide type, matching the legacy tuple above.""" + assert _param_basetype(cursor, prefix) == "nvarchar" + assert _param_basetype(cursor, "hello") == "varchar" + + +def test_native_unicode_kind_geometry_still_detected(cursor): + """Native path: a WKT string carrying a non-ASCII char is stored by CPython in a + wider (UCS-2/4) kind. Geometry detection must still fire — the old code gated the + prefix check on kind == 1BYTE and would have missed this, binding varchar.""" + tagged = "POLYGON((0 0,1 1,0 0)) café" + assert _param_basetype(cursor, tagged) == "nvarchar" + + +def test_native_large_geometry_binds_and_roundtrips(cursor): + """Native path: a >4000-char geometry WKT binds and round-trips. + + Regression guard for the geometry fix. Geometry now folds into the wide-type + decision but keeps the length-based DAE gate, so a large polygon streams as + NVARCHAR(MAX) via DAE. The earlier code that forced non-DAE columnSize == len + for geometry produced an unbindable NVARCHAR precision > 4000 ("Invalid precision + value"); see test_legacy_map_sql_type_large_geometry_is_unbindable for the shape + this deliberately avoids. + """ + ring = ",".join(f"{n} {n}" for n in range(900)) + ",900 0,0 0" + wkt = f"POLYGON((0 0,{ring}))" + assert len(wkt) > 4000 + cursor.execute("SELECT ?", [wkt]) + assert cursor.fetchone()[0] == wkt + # And SQL Server accepts it as real geometry, confirming the WKT arrived intact. + cursor.execute("SELECT geometry::STGeomFromText(?, 0).STAsText()", [wkt]) + assert cursor.fetchone()[0] + + +def test_legacy_map_sql_type_large_geometry_is_unbindable(cursor): + """Pins a known legacy defect so it stays visible: for a >4000-char polygon the + Python _map_sql_type returns SQL_WVARCHAR with columnSize == len and DAE=False. + That precision exceeds SQL Server's non-MAX NVARCHAR limit (4000) and is rejected + at bind time with "Invalid precision value". The native path deliberately does not + reproduce this shape — it streams via DAE instead. If legacy is ever fixed to gate + geometry on length, update this test. + """ + wkt = "POLYGON((" + ",".join(f"{n} {n}" for n in range(1000)) + "))" + assert len(wkt) > 4000 + sql_type, c_type, column_size, decimal_digits, is_dae = _detect(cursor, wkt) + assert sql_type == _c.SQL_WVARCHAR.value + assert column_size == len(wkt) > 4000 + assert is_dae is False + + +def test_map_sql_type_large_binary_uses_dae(cursor): + assert _detect(cursor, b"x" * 8001) == ( + _c.SQL_VARBINARY.value, + _c.SQL_C_BINARY.value, + 0, + 0, + True, + ) + + +def test_map_sql_type_aware_datetime(cursor): + aware = datetime.datetime( + 2024, 1, 1, 2, 3, 4, tzinfo=datetime.timezone(datetime.timedelta(hours=5, minutes=30)) + ) + assert _detect(cursor, aware) == ( + _c.SQL_DATETIMEOFFSET.value, + _c.SQL_C_SS_TIMESTAMPOFFSET.value, + 34, + 7, + False, + ) + + +@pytest.mark.parametrize( + "value", + [decimal.Decimal("100.50"), decimal.Decimal("214748.3647"), decimal.Decimal("214748.3648")], + ids=["smallmoney", "smallmoney-max", "money"], +) +def test_map_sql_type_money_range_binds_as_text(cursor, value): + """MONEY / SMALLMONEY range Decimals are formatted to text and the slot is + replaced with that formatted string.""" + params = [value] + sql_type, c_type, column_size, decimal_digits, is_dae = cursor._map_sql_type(value, params, 0) + assert (sql_type, c_type, decimal_digits, is_dae) == ( + _c.SQL_VARCHAR.value, + _c.SQL_C_CHAR.value, + 0, + False, + ) + assert params[0] == format(value, "f") + assert column_size == len(params[0]) + + +def test_map_sql_type_numeric_out_of_money_range(cursor): + """A Decimal beyond the MONEY range falls to the generic NUMERIC binding and the + slot is replaced with a NumericData struct.""" + value = decimal.Decimal("1E20") # 1e20 > MONEY_MAX (~9.2e14) + params = [value] + sql_type, c_type, column_size, decimal_digits, is_dae = cursor._map_sql_type(value, params, 0) + assert (sql_type, c_type, is_dae) == ( + _c.SQL_NUMERIC.value, + _c.SQL_C_NUMERIC.value, + False, + ) + assert column_size == params[0].precision + assert decimal_digits == params[0].scale + + +def test_map_sql_type_unsupported_raises_typeerror(cursor): + with pytest.raises(TypeError): + cursor._map_sql_type({1, 2, 3}, [{1, 2, 3}], 0) + + +# --------------------------------------------------------------------------- +# _get_numeric_data: precision/scale arithmetic and digit packing +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value, precision, scale", + [ + (decimal.Decimal("0"), 1, 0), + (decimal.Decimal("314E2"), 5, 0), # positive exponent + (decimal.Decimal("3.140"), 4, 3), # -exp <= num_digits + (decimal.Decimal("0.03140"), 5, 5), # -exp > num_digits (leading-zero pad) + ], + ids=["zero", "pos-exp", "frac", "leading-zeros"], +) +def test_get_numeric_data_precision_scale(cursor, value, precision, scale): + nd = cursor._get_numeric_data(value) + assert nd.precision == precision + assert nd.scale == scale + + +def test_get_numeric_data_precision_overflow(cursor): + with pytest.raises(ValueError, match="too high"): + cursor._get_numeric_data(decimal.Decimal("1" * 39))