diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ebfb81..f4381a38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), before; users should call `cursor.setinputsizes()` to work around this. ### Fixed +- **GH-745:** `executemany` auto-detect binds money-range `Decimal` + values as `SQL_NUMERIC` with a batch-wide precision/scale (still via + `SQL_C_CHAR` string values), so a comparison against a smaller numeric + column no longer overflows. SQL precision/scale stay on `columnSize` / + `decimalDigits`; the CHAR array stride uses a separate `bufferSize` + sized from the longest fixed-point encoding (e.g. `Decimal("1E-38")`), + so near-max precision values are not rejected by the array buffer. + The `setinputsizes` DECIMAL/NUMERIC string path uses the same buffer-width + split; buffer width is derived from text produced by the protected + conversion path so failed conversions still raise a sanitized `ValueError` + with row/column details (no raw MemoryError/RuntimeError leakage). - **GH-740:** A Python `Decimal` whose value falls in the SQL Server MONEY / SMALLMONEY range is now bound as `SQL_NUMERIC` with its own precision and scale on both `execute()` paths (native detection, and the legacy path reached when diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 1cb12cb4..a6774d55 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -29,8 +29,8 @@ DatabaseError, ) from mssql_python.row import Row -from mssql_python.perf_timer import perf_phase from mssql_python import get_settings +from mssql_python.perf_timer import perf_phase from mssql_python.parameter_helper import ( detect_and_convert_parameters, parse_pyformat_params, @@ -667,9 +667,9 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg - i: The index of the parameter in the list. - decimal_as_numeric: When True, bind a Decimal as SQL_NUMERIC regardless of value, skipping the MONEY/SMALLMONEY-range VARCHAR shortcut. The execute() - path sets this so a money-range Decimal compared against a numeric column - does not overflow (GH-740). executemany() leaves it False because it - string-binds Decimals for the whole batch (GH-503). + path and executemany() auto-detect path set this so a money-range Decimal + compared against a numeric column does not overflow (GH-740, GH-745). + setinputsizes DECIMAL/NUMERIC still string-binds (GH-503). Returns: - A tuple containing the SQL type, C type, column size, and decimal digits. """ @@ -805,11 +805,11 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg f"The maximum precision supported by SQL Server is 38, but got {precision}." ) - # Detect MONEY / SMALLMONEY range. Skipped on the execute() path - # (decimal_as_numeric=True), where a money-range Decimal must bind as - # SQL_NUMERIC so a comparison against a smaller numeric column returns no - # match instead of overflowing (GH-740). executemany keeps the VARCHAR - # shortcut because it string-binds Decimals for the batch (GH-503). + # Detect MONEY / SMALLMONEY range. Skipped when decimal_as_numeric=True + # (execute() and executemany auto-detect), where a money-range Decimal must + # bind as SQL_NUMERIC so a comparison against a smaller numeric column + # returns no match instead of overflowing (GH-740, GH-745). The + # setinputsizes DECIMAL path still string-binds (GH-503). if not decimal_as_numeric and SMALLMONEY_MIN <= param <= SMALLMONEY_MAX: logger.debug("_map_sql_type: DECIMAL -> SMALLMONEY - index=%d", i) # smallmoney @@ -1732,37 +1732,34 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # it will be unwrapped for parameter binding. This means you cannot # pass a tuple as a single parameter value (but SQL Server doesn't # support tuple types as parameter values anyway). - with perf_phase("py::execute::param_prep"): - if parameters: - # Check if single parameter is a nested container that should be unwrapped - # e.g., execute("SELECT ?", (value,)) vs execute("SELECT ?, ?", ((1, 2),)) - if isinstance(parameters, tuple) and len(parameters) == 1: - if isinstance(parameters[0], (tuple, list, dict)): - actual_params = parameters[0] - elif isinstance(parameters[0], Row): - # A Row (e.g. from fetchone()) is a sequence of column values. - # Normalize it to a tuple so the downstream binding logic, which - # only handles tuple/list/dict, can unwrap it into individual - # parameters instead of treating the whole Row as one value. - actual_params = tuple(parameters[0]) - else: - actual_params = parameters + if parameters: + # Check if single parameter is a nested container that should be unwrapped + # e.g., execute("SELECT ?", (value,)) vs execute("SELECT ?, ?", ((1, 2),)) + if isinstance(parameters, tuple) and len(parameters) == 1: + if isinstance(parameters[0], (tuple, list, dict)): + actual_params = parameters[0] + elif isinstance(parameters[0], Row): + # A Row (e.g. from fetchone()) is a sequence of column values. + # Normalize it to a tuple so the downstream binding logic, which + # only handles tuple/list/dict, can unwrap it into individual + # parameters instead of treating the whole Row as one value. + actual_params = tuple(parameters[0]) else: actual_params = parameters + else: + actual_params = parameters - # Skip detect_and_convert_parameters when re-executing the same SQL — - # the parameter style (qmark vs pyformat) won't change between calls. - if operation == self.last_executed_stmt and isinstance( - actual_params, (tuple, list) - ): - parameters = list(actual_params) - else: - operation, converted_params = detect_and_convert_parameters( - operation, actual_params - ) - parameters = list(converted_params) + # Skip detect_and_convert_parameters when re-executing the same SQL — + # the parameter style (qmark vs pyformat) won't change between calls. + if operation == self.last_executed_stmt and isinstance(actual_params, (tuple, list)): + parameters = list(actual_params) else: - parameters = [] + operation, converted_params = detect_and_convert_parameters( + operation, actual_params + ) + parameters = list(converted_params) + else: + parameters = [] # Getting encoding setting encoding_settings = self._get_encoding_settings() @@ -1799,6 +1796,7 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state ) else: ret = ddbc_bindings.DDBCSQLExecDirect(self.hstmt, operation) + # Check return code try: @@ -1809,27 +1807,24 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self._reset_cursor() raise - # Capture any diagnostic messages (SQL_SUCCESS_WITH_INFO, etc.) - with perf_phase("py::execute::diag_records"): - self._capture_diagnostics(ret) + self._capture_diagnostics(ret) self.last_executed_stmt = operation - with perf_phase("py::execute::post_execute"): - # Update rowcount after execution - # TODO: rowcount return code from SQL needs to be handled - self.rowcount = ddbc_bindings.DDBCSQLRowCount(self.hstmt) + # Update rowcount after execution + # TODO: rowcount return code from SQL needs to be handled + self.rowcount = ddbc_bindings.DDBCSQLRowCount(self.hstmt) - # Initialize description after execution - # After successful execution, initialize description if there are results - column_metadata = [] - try: - ddbc_bindings.DDBCSQLDescribeCol(self.hstmt, column_metadata) - self._initialize_description(column_metadata) - except Exception as e: # pylint: disable=broad-exception-caught - # If describe fails, it's likely there are no results (e.g., for INSERT) - self.description = None - self._column_sql_types = None + # Initialize description after execution + # After successful execution, initialize description if there are results + column_metadata = [] + try: + ddbc_bindings.DDBCSQLDescribeCol(self.hstmt, column_metadata) + self._initialize_description(column_metadata) + except Exception as e: # pylint: disable=broad-exception-caught + # If describe fails, it's likely there are no results (e.g., for INSERT) + self.description = None + self._column_sql_types = None # Reset rownumber for new result set (only for SELECT statements) if self.description: # If we have column descriptions, it's likely a SELECT @@ -2295,6 +2290,53 @@ def _transpose_rowwise_to_columnwise( return columnwise, row_count + @staticmethod + def _decimal_sql_precision_scale(value: decimal.Decimal) -> Tuple[int, int]: + """Return SQL NUMERIC (precision, scale) for a finite Decimal. + + Matches the precision/scale rules used by _map_sql_type / _get_numeric_data. + """ + decimal_as_tuple = value.as_tuple() + digits_tuple = decimal_as_tuple.digits + num_digits = len(digits_tuple) + exponent = decimal_as_tuple.exponent + if isinstance(exponent, str): + raise ValueError("Cannot bind non-finite Decimal (NaN/Infinity) as SQL NUMERIC") + if exponent >= 0: + precision = num_digits + exponent + scale = 0 + elif (-1 * exponent) <= num_digits: + precision = num_digits + scale = exponent * -1 + else: + precision = exponent * -1 + scale = exponent * -1 + return precision, scale + + def _batch_decimal_precision_scale(self, column) -> Tuple[int, int]: + """Derive one NUMERIC(precision, scale) that fits every Decimal in a column. + + Used by executemany so money-range Decimals can bind as SQL_NUMERIC with a + single batch-wide type (GH-745) without shrinking any row's digits. + + Non-finite Decimals (NaN/Infinity) raise ValueError rather than being + skipped. Callers must enforce SQL Server's precision limit (<= 38). + """ + max_scale = 0 + max_int_digits = 0 + found = False + for value in column: + if not isinstance(value, decimal.Decimal): + continue + # Propagate non-finite errors; do not silently skip them. + precision, scale = self._decimal_sql_precision_scale(value) + found = True + max_scale = max(max_scale, scale) + max_int_digits = max(max_int_digits, precision - scale) + if not found: + return 0, 0 + return max(max_int_digits + max_scale, 1), max_scale + def _compute_column_type(self, column): """ Determine representative value and integer min/max for a column. @@ -2323,6 +2365,11 @@ def _compute_column_type(self, column): max_decimal_formatted_len = 0 for v in non_nulls: if isinstance(v, decimal.Decimal): + # Non-finite Decimals have a string exponent ('n'/'N'/'F'); comparing + # that to int raises TypeError before the NUMERIC ValueError path. + # Reject early with the same message used by _decimal_sql_precision_scale. + if not v.is_finite(): + raise ValueError("Cannot bind non-finite Decimal (NaN/Infinity) as SQL NUMERIC") max_decimal_formatted_len = max(max_decimal_formatted_len, len(format(v, "f"))) if not sample_value: sample_value = v @@ -2338,7 +2385,7 @@ def _compute_column_type(self, column): # If length comparison fails, keep the current sample_value pass elif isinstance(v, decimal.Decimal) and isinstance(sample_value, decimal.Decimal): - # For Decimal objects, prefer the one that requires higher precision or scale + # Both values are finite (checked above). Prefer higher precision/scale. v_tuple = v.as_tuple() sample_tuple = sample_value.as_tuple() @@ -2478,6 +2525,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ) # Prepare parameter type information + # Columns configured via setinputsizes keep declared columnSize/decimalDigits + # through the post-conversion widen pass (bufferSize may still grow). + explicit_inputsize_cols = set() with perf_phase("py::executemany::param_type_detection"): for col_index in range(param_count): column = ( @@ -2489,6 +2539,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s if self._inputsizes and col_index < len(self._inputsizes): # Use explicitly set input sizes + explicit_inputsize_cols.add(col_index) sql_type, c_type, column_size, decimal_digits = self._inputsizes[col_index] # Default is_dae to False @@ -2509,12 +2560,25 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s is_dae = True # Sanitize precision/scale for numeric types + numeric_buffer_size = 0 if sql_type in ( ddbc_sql_const.SQL_DECIMAL.value, ddbc_sql_const.SQL_NUMERIC.value, ): column_size = max(1, min(int(column_size) if column_size > 0 else 18, 38)) decimal_digits = min(max(0, decimal_digits), column_size) + # Provisional SQL_C_CHAR stride: size only from values that + # are already Decimal. Do NOT convert non-Decimals here — + # that bypasses the protected conversion loop below and can + # leak MemoryError/RuntimeError (and value-bearing messages). + # After conversion, bufferSize is widened from the produced + # fixed-point text (same path that sanitizes failures). + max_encoded = 0 + for row in seq_of_parameters: + value = row[col_index] + if isinstance(value, decimal.Decimal): + max_encoded = max(max_encoded, len(format(value, "f"))) + numeric_buffer_size = max(max_encoded, column_size + 3, 1) # For binary data columns with mixed content, we need to find max size if sql_type in ( @@ -2545,6 +2609,8 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s paraminfo.columnSize = column_size paraminfo.decimalDigits = decimal_digits paraminfo.isDAE = is_dae + if numeric_buffer_size: + paraminfo.bufferSize = numeric_buffer_size # Ensure we never have SQL_C_DEFAULT (0) for C-type if paraminfo.paramCType == 0: @@ -2562,6 +2628,18 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s column ) + # GH-745: auto-detected Decimal columns bind as SQL_NUMERIC (skipping + # the money-range VARCHAR shortcut) so a money-range value compared + # against a smaller numeric column does not overflow. executemany still + # string-binds via SQL_C_CHAR below; setinputsizes DECIMAL stays on the + # GH-503 string path above. + # Only force NUMERIC when every non-NULL value in the column is Decimal; + # a heterogeneous column keeps the prior sample-driven path. + non_null_values = [v for v in column if v is not None] + decimal_as_numeric = bool(non_null_values) and all( + isinstance(v, decimal.Decimal) for v in non_null_values + ) + dummy_row = list(sample_row) paraminfo = self._create_parameter_types_list( sample_value, @@ -2570,6 +2648,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s col_index, min_val=min_val, max_val=max_val, + decimal_as_numeric=decimal_as_numeric, ) # GH-610: all-NULL columns now pass SQL_UNKNOWN_TYPE to C++, @@ -2587,9 +2666,26 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ddbc_sql_const.SQL_NUMERIC.value, ): paraminfo.paramCType = ddbc_sql_const.SQL_C_CHAR.value - # Ensure columnSize accommodates the longest string representation - if max_decimal_len > paraminfo.columnSize: - paraminfo.columnSize = max_decimal_len + # One NUMERIC(precision, scale) must fit every Decimal in the + # batch (GH-745). Sample-only precision/scale is not enough. + # columnSize is NUMERIC precision for SQLBindParameter, not a + # string buffer length — do not widen it with max_decimal_len. + batch_precision, batch_scale = self._batch_decimal_precision_scale(column) + if batch_precision > 38: + raise ValueError( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is 38, " + f"but got {batch_precision}." + ) + if batch_precision > paraminfo.columnSize: + paraminfo.columnSize = batch_precision + if batch_scale > paraminfo.decimalDigits: + paraminfo.decimalDigits = batch_scale + # SQL_C_CHAR array stride is separate from SQL precision. + # Fixed-point strings need room for sign, '.', and a leading + # zero (e.g. Decimal("1E-38") -> 40 chars with precision 38). + # Size from the longest encoded value in the batch. + paraminfo.bufferSize = max(max_decimal_len, 1) # Correct column size for Decimal columns sent as SQL_VARCHAR (GH-557). # The sample value's formatted string may be shorter than another @@ -2636,9 +2732,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s self.execute(operation, row) return - # Process parameters into column-wise format with possible type conversions - # First, convert any Decimal types as needed for NUMERIC/DECIMAL columns with perf_phase("py::executemany::param_conversion"): + # Process parameters into column-wise format with possible type conversions + # First, convert any Decimal types as needed for NUMERIC/DECIMAL columns processed_parameters = [] for row_index, row in enumerate(seq_of_parameters): processed_row = list(row) @@ -2690,14 +2786,64 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s raise ValueError(err_msg) from None processed_parameters.append(processed_row) - # Now transpose the processed parameters with perf_phase("py::executemany::param_processing"): + # Derive/widen SQL_C_CHAR bufferSize and (for auto-detect only) SQL + # NUMERIC precision/scale from text produced by the protected conversion. + # setinputsizes previously sized by converting independently (leaking raw + # MemoryError/RuntimeError); the auto-detect path already had a + # Decimal-only provisional size. Post-conversion strings are authoritative + # for precision on auto-detect (e.g. Decimal("1e15.00") + "2e16"). Explicit + # setinputsizes columnSize/decimalDigits stay as declared; bufferSize still + # grows so the CHAR array fits. + for col_index, ptype in enumerate(parameters_type): + if ptype.paramSQLType not in ( + ddbc_sql_const.SQL_DECIMAL.value, + ddbc_sql_const.SQL_NUMERIC.value, + ): + continue + max_encoded = 0 + max_scale = 0 + max_int_digits = 0 + found_numeric_text = False + for row in processed_parameters: + val = row[col_index] + if not isinstance(val, str): + continue + max_encoded = max(max_encoded, len(val)) + try: + as_decimal = decimal.Decimal(val) + except decimal.DecimalException: + continue + if not as_decimal.is_finite(): + continue + precision, scale = self._decimal_sql_precision_scale(as_decimal) + found_numeric_text = True + max_scale = max(max_scale, scale) + max_int_digits = max(max_int_digits, precision - scale) + if max_encoded: + prior = getattr(ptype, "bufferSize", 0) or 0 + ptype.bufferSize = max(prior, max_encoded, 1) + # Honor explicit setinputsizes precision/scale; only auto-detect widens. + if found_numeric_text and col_index not in explicit_inputsize_cols: + batch_precision = max(max_int_digits + max_scale, 1) + if batch_precision > 38: + raise ValueError( + "Precision of the numeric value is too high. " + "The maximum precision supported by SQL Server is 38, " + f"but got {batch_precision}." + ) + if batch_precision > ptype.columnSize: + ptype.columnSize = batch_precision + if max_scale > ptype.decimalDigits: + ptype.decimalDigits = max_scale + + # Now transpose the processed parameters columnwise_params, row_count = self._transpose_rowwise_to_columnwise( processed_parameters ) - # Get encoding settings - encoding_settings = self._get_encoding_settings() + # Get encoding settings + encoding_settings = self._get_encoding_settings() # Debug logging: emit batch metadata only. Never log parameter values or # row representations here -- rows may contain PII (SSNs, emails, @@ -2721,9 +2867,8 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ) # Capture any diagnostic messages after execution - with perf_phase("py::executemany::diag_records"): - if self.hstmt: - self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) + if self.hstmt: + self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) try: check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) @@ -2778,18 +2923,16 @@ def fetchone(self) -> Union[None, Row]: # Fetch raw data row_data = [] try: - with perf_phase("py::fetchone::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchOne( - self.hstmt, - row_data, - char_decoding.get("encoding", "utf-16le"), - wchar_decoding.get("encoding", "utf-16le"), - char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), - ) + ret = ddbc_bindings.DDBCSQLFetchOne( + self.hstmt, + row_data, + char_decoding.get("encoding", "utf-16le"), + wchar_decoding.get("encoding", "utf-16le"), + char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), + ) - with perf_phase("py::fetchone::diag_records"): - if self.hstmt: - self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) + if self.hstmt: + self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) if ret == ddbc_sql_const.SQL_NO_DATA.value: # No more data available @@ -2809,18 +2952,17 @@ def fetchone(self) -> Union[None, Row]: # Get column and converter maps column_map, converter_map, column_map_lower = self._get_column_and_converter_maps() - with perf_phase("py::fetchone::row_wrap"): - return Row( - row_data, - column_map, - cursor=self, - converter_map=converter_map, - uuid_str_indices=self._uuid_str_indices, - column_map_lower=column_map_lower, - ) - except Exception: + return Row( + row_data, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=self._uuid_str_indices, + column_map_lower=column_map_lower, + ) + except Exception as e: # On error, don't increment rownumber - rethrow the error - raise + raise e def fetchmany(self, size: Optional[int] = None) -> List[Row]: """ @@ -2848,19 +2990,17 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: # Fetch raw data rows_data = [] try: - with perf_phase("py::fetchmany::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchMany( - self.hstmt, - rows_data, - size, - char_decoding.get("encoding", "utf-16le"), - wchar_decoding.get("encoding", "utf-16le"), - char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), - ) + ret = ddbc_bindings.DDBCSQLFetchMany( + self.hstmt, + rows_data, + size, + char_decoding.get("encoding", "utf-16le"), + wchar_decoding.get("encoding", "utf-16le"), + char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), + ) - with perf_phase("py::fetchmany::diag_records"): - if self.hstmt: - self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) + if self.hstmt: + self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) # Update rownumber for the number of rows actually fetched if rows_data and self._has_result_set: @@ -2879,21 +3019,20 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]: # Convert raw data to Row objects uuid_idx = self._uuid_str_indices - with perf_phase("py::fetchmany::row_wrap"): - return [ - Row( - row_data, - column_map, - cursor=self, - converter_map=converter_map, - uuid_str_indices=uuid_idx, - column_map_lower=column_map_lower, - ) - for row_data in rows_data - ] - except Exception: + return [ + Row( + row_data, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=uuid_idx, + column_map_lower=column_map_lower, + ) + for row_data in rows_data + ] + except Exception as e: # On error, don't increment rownumber - rethrow the error - raise + raise e def fetchall(self) -> List[Row]: """ @@ -2912,21 +3051,19 @@ def fetchall(self) -> List[Row]: # Fetch raw data rows_data = [] try: - with perf_phase("py::fetchall::cpp_call"): - ret = ddbc_bindings.DDBCSQLFetchAll( - self.hstmt, - rows_data, - char_decoding.get("encoding", "utf-16le"), - wchar_decoding.get("encoding", "utf-16le"), - char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), - ) + ret = ddbc_bindings.DDBCSQLFetchAll( + self.hstmt, + rows_data, + char_decoding.get("encoding", "utf-16le"), + wchar_decoding.get("encoding", "utf-16le"), + char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value), + ) # Check for errors check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - with perf_phase("py::fetchall::diag_records"): - if self.hstmt: - self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) + if self.hstmt: + self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt)) # Update rownumber for the number of rows actually fetched if rows_data and self._has_result_set: @@ -2944,21 +3081,20 @@ def fetchall(self) -> List[Row]: # Convert raw data to Row objects uuid_idx = self._uuid_str_indices - with perf_phase("py::fetchall::row_wrap"): - return [ - Row( - row_data, - column_map, - cursor=self, - converter_map=converter_map, - uuid_str_indices=uuid_idx, - column_map_lower=column_map_lower, - ) - for row_data in rows_data - ] - except Exception: + return [ + Row( + row_data, + column_map, + cursor=self, + converter_map=converter_map, + uuid_str_indices=uuid_idx, + column_map_lower=column_map_lower, + ) + for row_data in rows_data + ] + except Exception as e: # On error, don't increment rownumber - rethrow the error - raise + raise e def arrow_batch(self, batch_size: int = 8192) -> "pyarrow.RecordBatch": """ diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 26948ee8..d9009d01 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2197,6 +2197,22 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, return exec_rc; } +// Array element width for SQL_C_CHAR / SQL_C_WCHAR / SQL_C_BINARY parameter +// arrays. columnSize is the SQLBindParameter ColumnSize (NUMERIC precision for +// SQL_NUMERIC/SQL_DECIMAL). bufferSize, when set, is the encoded-string stride. +// For NUMERIC/DECIMAL string binds with bufferSize==0, reserve precision+3 so +// sign, decimal point, and a leading zero always fit (e.g. "-0." + 38 digits). +static inline SQLULEN ArrayDataBufferWidth(const ParamInfo& info) { + if (info.bufferSize > 0) { + return info.bufferSize; + } + if ((info.paramSQLType == SQL_NUMERIC || info.paramSQLType == SQL_DECIMAL) && + (info.paramCType == SQL_C_CHAR || info.paramCType == SQL_C_WCHAR)) { + return info.columnSize + 3; + } + return info.columnSize; +} + SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& columnwise_params, std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers, @@ -2269,27 +2285,28 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& break; } case SQL_C_WCHAR: { + const SQLULEN dataWidth = ArrayDataBufferWidth(info); LOG("BindParameterArray: Binding SQL_C_WCHAR array - " - "param_index=%d, count=%zu, column_size=%zu", - paramIndex, paramSetSize, info.columnSize); + "param_index=%d, count=%zu, column_size=%zu, buffer_width=%zu", + paramIndex, paramSetSize, info.columnSize, dataWidth); SQLWCHAR* wcharArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, paramSetSize * (dataWidth + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(wcharArray + i * (info.columnSize + 1), 0, - (info.columnSize + 1) * sizeof(SQLWCHAR)); + std::memset(wcharArray + i * (dataWidth + 1), 0, + (dataWidth + 1) * sizeof(SQLWCHAR)); } else { std::u16string wstr = columnValues[i].cast(); // u16string is already UTF-16, so the // original check is sufficient - if (wstr.length() > info.columnSize) { + if (wstr.length() > dataWidth) { ThrowStdException("Input string exceeds allowed column size " "at parameter index " + std::to_string(paramIndex)); } - std::memcpy(wcharArray + i * (info.columnSize + 1), wstr.c_str(), + std::memcpy(wcharArray + i * (dataWidth + 1), wstr.c_str(), (wstr.length() + 1) * sizeof(SQLWCHAR)); strLenOrIndArray[i] = SQL_NTS; } @@ -2298,7 +2315,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = wcharArray; - bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); + bufferLength = (dataWidth + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: @@ -2366,17 +2383,23 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& } case SQL_C_CHAR: case SQL_C_BINARY: { + // dataWidth is the per-row buffer stride. For SQL_NUMERIC / + // SQL_DECIMAL + SQL_C_CHAR, columnSize stays as SQL precision + // for SQLBindParameter; buffer width comes from bufferSize + // (longest fixed-point encoding) or precision+3 fallback. + const SQLULEN dataWidth = ArrayDataBufferWidth(info); LOG("BindParameterArray: Binding SQL_C_CHAR/BINARY array - " - "param_index=%d, count=%zu, column_size=%zu, encoding='%s'", - paramIndex, paramSetSize, info.columnSize, charEncoding.c_str()); + "param_index=%d, count=%zu, column_size=%zu, buffer_width=%zu, " + "encoding='%s'", + paramIndex, paramSetSize, info.columnSize, dataWidth, + charEncoding.c_str()); char* charArray = AllocateParamBufferArray( - tempBuffers, paramSetSize * (info.columnSize + 1)); + tempBuffers, paramSetSize * (dataWidth + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; - std::memset(charArray + i * (info.columnSize + 1), 0, - info.columnSize + 1); + std::memset(charArray + i * (dataWidth + 1), 0, dataWidth + 1); } else { std::string encodedStr; @@ -2406,15 +2429,15 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& encodedStr = columnValues[i].cast(); } - if (encodedStr.size() > info.columnSize) { + if (encodedStr.size() > dataWidth) { LOG("BindParameterArray: String/binary too " "long - param_index=%d, row=%zu, size=%zu, " "max=%zu", - paramIndex, i, encodedStr.size(), info.columnSize); + paramIndex, i, encodedStr.size(), dataWidth); ThrowStdException("Input exceeds column size at index " + std::to_string(i)); } - std::memcpy(charArray + i * (info.columnSize + 1), encodedStr.c_str(), + std::memcpy(charArray + i * (dataWidth + 1), encodedStr.c_str(), encodedStr.size()); strLenOrIndArray[i] = static_cast(encodedStr.size()); } @@ -2423,7 +2446,7 @@ SQLRETURN BindParameterArray(SqlHandle& handle, SQLHANDLE hStmt, const py::list& "param_index=%d", paramIndex); dataPtr = charArray; - bufferLength = info.columnSize + 1; + bufferLength = static_cast(dataWidth + 1); break; } case SQL_C_BIT: { @@ -6054,6 +6077,7 @@ PYBIND11_MODULE(ddbc_bindings, m) { .def_readwrite("paramCType", &ParamInfo::paramCType) .def_readwrite("paramSQLType", &ParamInfo::paramSQLType) .def_readwrite("columnSize", &ParamInfo::columnSize) + .def_readwrite("bufferSize", &ParamInfo::bufferSize) .def_readwrite("decimalDigits", &ParamInfo::decimalDigits) .def_readwrite("strLenOrInd", &ParamInfo::strLenOrInd) .def_property( diff --git a/mssql_python/pybind/param_detect.hpp b/mssql_python/pybind/param_detect.hpp index 172a98b5..cb779d71 100644 --- a/mssql_python/pybind/param_detect.hpp +++ b/mssql_python/pybind/param_detect.hpp @@ -51,6 +51,13 @@ struct ParamInfo { SQLSMALLINT paramCType = SQL_C_DEFAULT; SQLSMALLINT paramSQLType = SQL_UNKNOWN_TYPE; SQLULEN columnSize = 0; + // Character/binary array element width for BindParameterArray (bytes for + // SQL_C_CHAR/BINARY, code units for SQL_C_WCHAR). Distinct from columnSize: + // for SQL_NUMERIC/SQL_DECIMAL, columnSize is SQL precision (digit count) + // while the SQL_C_CHAR buffer must also fit sign, decimal point, leading + // zero, and the encoded digits (e.g. Decimal("1E-38") -> 40 chars). + // 0 means "derive from columnSize" (with a NUMERIC/DECIMAL fallback). + SQLULEN bufferSize = 0; SQLSMALLINT decimalDigits = 0; SQLLEN strLenOrInd = 0; // Required for DAE bool isDAE = false; // Indicates if we need to stream diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index 219d4833..50f52209 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -10558,20 +10558,21 @@ def test_setinputsizes_sql_decimal_unconvertible_value(db_connection): def test_setinputsizes_sql_decimal_str_raises_no_leak(db_connection): - """A parameter whose str() raises must not leak the exception text (GH-503). + """A parameter whose str() raises RuntimeError must not leak the text (GH-503). Exception chaining (raise ... from e) can surface a value-bearing cause through __cause__ and formatted tracebacks. For a value whose str() raises, the chain must be suppressed so the metadata-only guarantee holds across - tracebacks and APM/log shippers, not just str(exc). + tracebacks and APM/log shippers, not just str(exc). The sizing pass must + not convert independently, or a raw RuntimeError escapes sanitization. """ cursor = db_connection.cursor() - secret = "secret-987-65-4321" + secret = "synthetic-private-parameter" class ExplodingStr: def __str__(self): - raise ValueError(secret) + raise RuntimeError(secret) cursor.execute("DROP TABLE IF EXISTS #test_sis_dec_explode") try: @@ -17234,6 +17235,505 @@ def test_map_sql_type_decimal_in_money_returns_varchar(): assert c_type == _C.SQL_C_CHAR.value +def test_gh745_batch_decimal_precision_scale_covers_all_rows(): + """_batch_decimal_precision_scale fits every Decimal in the column.""" + cur = _make_bare_cursor() + column = [ + decimal.Decimal("1.0"), + decimal.Decimal("12345.6789"), + decimal.Decimal("-0.1"), + ] + precision, scale = cur._batch_decimal_precision_scale(column) + assert scale >= 4 + assert precision >= 9 + + +def test_gh745_executemany_money_range_binds_as_numeric(monkeypatch): + """executemany auto-detect binds money-range Decimals as SQL_NUMERIC (GH-745).""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 2) + data = [ + (decimal.Decimal("12.34"),), + (decimal.Decimal("12345.6789"),), + (decimal.Decimal("-0.1"),), + ] + cur.executemany("UPDATE t SET x = 1 WHERE v = ?", data) + pt = captured["parameters_type"] + assert len(pt) == 1 + assert pt[0].paramSQLType == _C.SQL_NUMERIC.value + assert pt[0].paramCType == _C.SQL_C_CHAR.value + # columnSize is NUMERIC precision (not formatted-string length). + assert pt[0].columnSize == 9 + assert pt[0].columnSize <= 38 + assert pt[0].decimalDigits >= 4 + longest = max(len(v) for v in captured["columnwise_params"][0]) + assert pt[0].bufferSize >= longest + for val in captured["columnwise_params"][0]: + assert isinstance(val, str) + + +def test_gh745_batch_decimal_rejects_non_finite(): + """_batch_decimal_precision_scale must not silently skip NaN/Infinity.""" + cur = _make_bare_cursor() + column = [decimal.Decimal("1.0"), decimal.Decimal("NaN")] + with pytest.raises(ValueError, match="non-finite"): + cur._batch_decimal_precision_scale(column) + + +def test_gh745_executemany_near_max_precision_stays_within_38(monkeypatch): + """NUMERIC columnSize stays <= 38; CHAR bufferWidth fits Decimal("1E-38").""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 1) + # Decimal('1E-38') needs precision=38, scale=38. Formatted string length is > 38, + # so columnSize must stay at precision while bufferSize covers the encoding. + tiny = decimal.Decimal("1E-38") + encoded = format(tiny, "f") + data = [(tiny,)] + cur.executemany("UPDATE t SET x = 1 WHERE v = ?", data) + pt = captured["parameters_type"][0] + assert pt.paramSQLType == _C.SQL_NUMERIC.value + assert pt.columnSize == 38 + assert pt.decimalDigits == 38 + assert len(encoded) == 40 + assert pt.bufferSize >= len(encoded) + assert captured["columnwise_params"][0][0] == encoded + + +def test_gh745_executemany_buffer_fits_mixed_sign_short_precision(monkeypatch): + """Mixed-sign Decimals must fit CHAR buffer even when precision is small. + + e.g. [-12.34, 56.78] -> NUMERIC(4,2) but "-12.34" is 6 characters. + """ + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 2) + data = [(decimal.Decimal("-12.34"),), (decimal.Decimal("56.78"),)] + cur.executemany("UPDATE t SET x = 1 WHERE v = ?", data) + pt = captured["parameters_type"][0] + encoded = [format(decimal.Decimal("-12.34"), "f"), format(decimal.Decimal("56.78"), "f")] + assert pt.paramSQLType == _C.SQL_NUMERIC.value + assert pt.columnSize == 4 + assert pt.decimalDigits == 2 + assert pt.bufferSize >= max(len(s) for s in encoded) + assert pt.bufferSize > pt.columnSize + assert captured["columnwise_params"][0] == encoded + + +def test_setinputsizes_sql_decimal_memoryerror_no_leak_unit(monkeypatch): + """Huge scientific string must raise sanitized ValueError, not MemoryError.""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", lambda *a, **k: 0) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 0) + + sensitive_value = "1e999999999999999999" + cur.setinputsizes([(mssql_python.SQL_DECIMAL, 18, 2)]) + with pytest.raises(ValueError) as exc_info: + cur.executemany("INSERT INTO t VALUES (?)", [(sensitive_value,)]) + + message = str(exc_info.value) + assert "Failed to convert parameter" in message + assert "row 0" in message + assert "column 0" in message + assert exc_info.value.__cause__ is None + assert sensitive_value not in message + formatted = "".join( + traceback.format_exception( + type(exc_info.value), exc_info.value, exc_info.value.__traceback__ + ) + ) + assert sensitive_value not in formatted + + +def test_setinputsizes_sql_decimal_runtimeerror_no_leak_unit(monkeypatch): + """str() raising RuntimeError must become sanitized ValueError (no marker leak).""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", lambda *a, **k: 0) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 0) + + marker = "synthetic-private-parameter" + + class ExplodingStr: + def __str__(self): + raise RuntimeError(marker) + + cur.setinputsizes([(mssql_python.SQL_DECIMAL, 18, 2)]) + with pytest.raises(ValueError) as exc_info: + cur.executemany("INSERT INTO t VALUES (?)", [(ExplodingStr(),)]) + + assert marker not in str(exc_info.value) + assert exc_info.value.__cause__ is None + assert "row 0" in str(exc_info.value) + assert "column 0" in str(exc_info.value) + formatted = "".join( + traceback.format_exception( + type(exc_info.value), exc_info.value, exc_info.value.__traceback__ + ) + ) + assert marker not in formatted + + +def test_setinputsizes_sql_numeric_keeps_declared_precision_scale(monkeypatch): + """Explicit setinputsizes NUMERIC(10,2) must not widen after conversion (sumitmsft). + + bufferSize may still grow to fit the encoded text, but columnSize/decimalDigits + stay at the declared (10, 2) even when Decimal("1.234") needs scale 3. + """ + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 1) + + cur.setinputsizes([(mssql_python.SQL_NUMERIC, 10, 2)]) + cur.executemany("SELECT ?", [(decimal.Decimal("1.234"),)]) + pt = captured["parameters_type"][0] + encoded = format(decimal.Decimal("1.234"), "f") + assert pt.paramSQLType == _C.SQL_NUMERIC.value + assert pt.columnSize == 10 + assert pt.decimalDigits == 2 + assert pt.bufferSize >= len(encoded) + assert captured["columnwise_params"][0][0] == encoded + + +def test_setinputsizes_sql_decimal_buffer_from_protected_conversion(monkeypatch): + """setinputsizes DECIMAL bufferSize comes from protected conversion text. + + Provisional sizing must not convert non-Decimals (that leaked MemoryError / + RuntimeError). After the protected loop, bufferSize must still fit + Decimal("1E-38") and string inputs like "1E-38". + """ + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + def _run(data): + cur = Cursor.__new__(Cursor) + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: len(data)) + cur.setinputsizes([(mssql_python.SQL_DECIMAL, 38, 38)]) + cur.executemany("INSERT INTO t VALUES (?)", data) + return captured + + tiny = decimal.Decimal("1E-38") + encoded = format(tiny, "f") + assert len(encoded) == 40 + + for payload in ([(tiny,)], [("1E-38",)]): + captured = _run(payload) + pt = captured["parameters_type"][0] + assert pt.paramSQLType == _C.SQL_DECIMAL.value + assert pt.columnSize == 38 + assert pt.decimalDigits == 38 + assert pt.bufferSize >= len(encoded) + assert captured["columnwise_params"][0][0] == encoded + + +def test_gh745_executemany_batch_precision_over_38_raises(monkeypatch): + """Mixed batch whose combined precision exceeds 38 must raise ValueError.""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", lambda *a, **k: 0) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 0) + # 20 integer digits + 20 fractional digits across rows => batch precision 40. + data = [ + (decimal.Decimal("1" * 20),), + (decimal.Decimal("0." + ("1" * 20)),), + ] + with pytest.raises(ValueError, match="maximum precision supported by SQL Server is 38"): + cur.executemany("UPDATE t SET x = 1 WHERE v = ?", data) + + +def test_gh745_executemany_mixed_decimal_string_widens_precision(monkeypatch): + """Post-conversion numeric strings must widen NUMERIC columnSize (bewithgaurav). + + Decimal("1000000000000000.00") alone is NUMERIC(18,2) (16 integer digits). A + sibling string "20000000000000000" needs 17 integer digits. After the protected + conversion, columnSize must cover both without using formatted-string length + as SQL precision. + """ + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + captured["columnwise_params"] = col_params + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 2) + + decimal_val = decimal.Decimal("1000000000000000.00") + string_val = "20000000000000000" + data = [(decimal_val,), (string_val,)] + cur.executemany("INSERT INTO t VALUES (?)", data) + + pt = captured["parameters_type"][0] + encoded = [format(decimal_val, "f"), format(decimal.Decimal(string_val), "f")] + assert pt.paramSQLType == _C.SQL_NUMERIC.value + assert pt.paramCType == _C.SQL_C_CHAR.value + # 17 integer digits (from the string) + scale 2 (from the Decimal) => 19. + # Do not treat formatted-string length as precision; it only coincides here. + assert pt.columnSize == 19 + assert pt.columnSize <= 38 + assert pt.decimalDigits == 2 + assert pt.bufferSize >= max(len(s) for s in encoded) + assert captured["columnwise_params"][0] == encoded + + +@pytest.mark.parametrize( + "data", + [ + [(decimal.Decimal("1.0"),), (decimal.Decimal("NaN"),)], + [(decimal.Decimal("NaN"),), (decimal.Decimal("1.0"),)], + [(decimal.Decimal("Infinity"),), (decimal.Decimal("1.0"),)], + [(decimal.Decimal("1.0"),), (decimal.Decimal("-Infinity"),)], + ], + ids=["finite-then-nan", "nan-then-finite", "inf-then-finite", "finite-then-neginf"], +) +def test_gh745_executemany_rejects_non_finite_both_orders(monkeypatch, data): + """executemany must raise ValueError for NaN/Inf, not TypeError (sumitmsft).""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", lambda *a, **k: 0) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 0) + + with pytest.raises(ValueError, match="non-finite"): + cur.executemany("INSERT INTO t VALUES (?)", data) + + +def test_gh745_compute_column_type_rejects_non_finite(): + """_compute_column_type must reject NaN before exponent comparisons.""" + cur = _make_bare_cursor() + with pytest.raises(ValueError, match="non-finite"): + cur._compute_column_type([decimal.Decimal("1.0"), decimal.Decimal("NaN")]) + + +def test_gh745_executemany_heterogeneous_column_skips_numeric_force(monkeypatch): + """A Decimal sample plus a non-Decimal value must not force the NUMERIC path.""" + from unittest.mock import MagicMock + from mssql_python import ddbc_bindings + from mssql_python.cursor import Cursor + + cur = Cursor.__new__(Cursor) + cur._inputsizes = None + cur._timeout = 0 + cur.closed = False + cur.hstmt = MagicMock() + cur.messages = [] + cur.is_stmt_prepared = [False] + cur._connection = MagicMock() + cur._connection._encoding = "utf-8" + cur._connection._conn = MagicMock() + captured = {} + + def fake_sql_execute_many(hstmt, op, col_params, param_types, row_count, enc): + captured["parameters_type"] = param_types + return 0 + + monkeypatch.setattr(cur, "_check_closed", lambda: None) + monkeypatch.setattr(cur, "_reset_cursor", lambda: None) + monkeypatch.setattr(ddbc_bindings, "SQLExecuteMany", fake_sql_execute_many) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLGetAllDiagRecords", lambda h: []) + monkeypatch.setattr(ddbc_bindings, "DDBCSQLRowCount", lambda h: 2) + data = [ + (decimal.Decimal("12.34"),), + ("not-a-decimal",), + ] + cur.executemany("UPDATE t SET x = 1 WHERE v = ?", data) + pt = captured["parameters_type"][0] + # Sample is Decimal but column is heterogeneous — stay off the forced NUMERIC path. + assert pt.paramSQLType != _C.SQL_NUMERIC.value + + def test_executemany_numeric_override_needed(): """The executemany auto-detection path must override SQL_C_NUMERIC to SQL_C_CHAR (GH-609).""" from mssql_python import ddbc_bindings diff --git a/tests/test_020_money_smallmoney.py b/tests/test_020_money_smallmoney.py index f60d37e0..688698d6 100644 --- a/tests/test_020_money_smallmoney.py +++ b/tests/test_020_money_smallmoney.py @@ -8,8 +8,9 @@ SQL_NUMERIC using its own precision and scale, regardless of value. Binding no longer depends on whether the value falls in the MONEY/SMALLMONEY range, so an in-range value compared against a smaller numeric column returns no match instead of a varchar->numeric -overflow (GH-740). executemany still string-binds Decimals (SQL_VARCHAR) to preserve -scale-38 precision (GH-503), so that path is unchanged here. +overflow (GH-740). executemany auto-detect likewise binds Decimals as SQL_NUMERIC with +a batch-wide precision/scale and SQL_C_CHAR string values (GH-745); setinputsizes +DECIMAL/NUMERIC still string-binds for fixed precision (GH-503). """ import pytest @@ -812,3 +813,125 @@ def test_gh740_signed_zero_normalizes(cursor, db_connection): finally: drop_table_if_exists(cursor, table_name) db_connection.commit() + + +# ============================================================================= +# GH-745: executemany money-range Decimal must bind as SQL_NUMERIC, not VARCHAR +# ============================================================================= + + +def test_gh745_executemany_in_range_decimal_numeric_comparison_no_overflow(cursor, db_connection): + """executemany must not overflow money-range Decimals against a smaller numeric. + + Before the fix, executemany still used the MONEY-range VARCHAR shortcut, so + SQL Server did a varchar->numeric conversion that overflowed instead of simply + not matching (the execute() path was fixed in GH-740 / #742). + """ + table_name = "#pytest_gh745_cmp" + try: + drop_table_if_exists(cursor, table_name) + cursor.execute(f"CREATE TABLE {table_name} (v numeric(5,2))") # max 999.99 + cursor.execute(f"INSERT INTO {table_name} VALUES (?)", [Decimal("12.34")]) + db_connection.commit() + + # Comparison via executemany is an unnatural shape, but it is the path that + # still carried the VARCHAR shortcut. UPDATE ... WHERE keeps the binding. + cursor.executemany( + f"UPDATE {table_name} SET v = v WHERE v = ?", + [(Decimal("12345.6789"),), (Decimal("300000.00"),)], + ) + cursor.execute(f"SELECT COUNT(*) FROM {table_name}") + assert cursor.fetchone()[0] == 1 + + cursor.executemany( + f"UPDATE {table_name} SET v = v WHERE v = ?", + [(Decimal("12.34"),)], + ) + cursor.execute(f"SELECT COUNT(*) FROM {table_name} WHERE v = ?", [Decimal("12.34")]) + assert cursor.fetchone()[0] == 1 + finally: + drop_table_if_exists(cursor, table_name) + db_connection.commit() + + +def test_gh745_executemany_mixed_sign_money_range_batch(cursor, db_connection): + """Mixed-sign money-range Decimals still insert through executemany (GH-557).""" + table_name = "#pytest_gh745_sign" + try: + drop_table_if_exists(cursor, table_name) + cursor.execute(f"CREATE TABLE {table_name} (v DECIMAL(28, 14))") + data = [ + (Decimal("1.0"),), + (Decimal("-0.1"),), + (Decimal("100.5"),), + (Decimal("-999.99"),), + ] + cursor.executemany(f"INSERT INTO {table_name} VALUES (?)", data) + db_connection.commit() + cursor.execute(f"SELECT COUNT(*) FROM {table_name}") + assert cursor.fetchone()[0] == 4 + finally: + drop_table_if_exists(cursor, table_name) + db_connection.commit() + + +def test_gh745_executemany_tiny_scale38_roundtrip(cursor, db_connection): + """executemany must accept Decimal("1E-38") into numeric(38,38). + + SQL precision stays 38; the SQL_C_CHAR array buffer must be wider than + precision because format(Decimal("1E-38"), "f") is 40 characters. + """ + table_name = "#pytest_gh745_tiny" + value = Decimal("1E-38") + try: + drop_table_if_exists(cursor, table_name) + cursor.execute(f"CREATE TABLE {table_name} (v numeric(38,38))") + cursor.executemany(f"INSERT INTO {table_name} VALUES (?)", [(value,), (Decimal("-1E-38"),)]) + db_connection.commit() + + cursor.execute(f"SELECT v FROM {table_name} ORDER BY v") + rows = [r[0] for r in cursor.fetchall()] + assert rows[0].as_tuple() == Decimal("-1E-38").as_tuple() + assert rows[1].as_tuple() == value.as_tuple() + finally: + drop_table_if_exists(cursor, table_name) + db_connection.commit() + + +def test_gh745_executemany_mixed_decimal_string_precision(cursor, db_connection): + """Decimal + numeric-string batch must fit DECIMAL(38,14) (bewithgaurav). + + Auto-detect used to derive NUMERIC(18,2) from the Decimal alone, then fail + the 17-digit string with DataError even though the destination fits both. + """ + table_name = "#pytest_gh745_mixed_prec" + try: + drop_table_if_exists(cursor, table_name) + cursor.execute(f"CREATE TABLE {table_name} (v DECIMAL(38, 14))") + data = [ + (Decimal("1000000000000000.00"),), + ("20000000000000000",), + ] + cursor.executemany(f"INSERT INTO {table_name} VALUES (?)", data) + db_connection.commit() + cursor.execute(f"SELECT v FROM {table_name} ORDER BY v") + rows = [r[0] for r in cursor.fetchall()] + assert rows[0] == Decimal("1000000000000000.00") + assert rows[1] == Decimal("20000000000000000") + finally: + drop_table_if_exists(cursor, table_name) + db_connection.commit() + + +@pytest.mark.parametrize( + "data", + [ + [(Decimal("1.0"),), (Decimal("NaN"),)], + [(Decimal("NaN"),), (Decimal("1.0"),)], + ], + ids=["finite-then-nan", "nan-then-finite"], +) +def test_gh745_executemany_rejects_nan_both_orders(cursor, data): + """executemany raises ValueError for NaN in either row order (sumitmsft).""" + with pytest.raises(ValueError, match="non-finite"): + cursor.executemany("INSERT INTO #unused_nan_table VALUES (?)", data)