Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/include/postgres_binary_file_reader.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ class PostgresBinaryFileReader {
static constexpr idx_t COPY_FILE_HEADER_SIZE = PostgresConversion::COPY_HEADER_LENGTH + 8;

PostgresBinaryFileReader(ClientContext &context, const string &file_path, vector<LogicalType> types,
vector<PostgresType> postgres_types, idx_t buffer_size = DEFAULT_BUFFER_SIZE);
vector<PostgresType> postgres_types, PostgresTypeConfig type_config,
idx_t buffer_size = DEFAULT_BUFFER_SIZE);

bool ReadChunk(DataChunk &output);

Expand Down
24 changes: 20 additions & 4 deletions src/include/postgres_binary_parser.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

class PostgresBinaryParser {
public:
PostgresBinaryParser(vector<LogicalType> types, vector<PostgresType> postgres_types);
PostgresBinaryParser(vector<LogicalType> types, vector<PostgresType> postgres_types,
PostgresTypeConfig type_config);

void SetBuffer(data_ptr_t buf, idx_t len);
bool ReadChunk(DataChunk &output, const vector<column_t> &column_ids);
Expand All @@ -40,6 +41,7 @@

vector<LogicalType> types;
vector<PostgresType> postgres_types;
PostgresTypeConfig type_config;

private:
template <class T>
Expand All @@ -52,7 +54,7 @@
} else if (sizeof(T) == sizeof(uint32_t)) {
val = ntohl(val);
} else if (sizeof(T) == sizeof(uint64_t)) {
val = ntohll(val);

Check warning on line 57 in src/include/postgres_binary_parser.hpp

View workflow job for this annotation

GitHub Actions / Windows (pg-17)

'>>': shift count negative or too big, undefined behavior

Check warning on line 57 in src/include/postgres_binary_parser.hpp

View workflow job for this annotation

GitHub Actions / Windows (pg-17)

'>>': shift count negative or too big, undefined behavior

Check warning on line 57 in src/include/postgres_binary_parser.hpp

View workflow job for this annotation

GitHub Actions / Windows (pg-17)

'>>': shift count negative or too big, undefined behavior
} else {
D_ASSERT(0);
}
Expand Down Expand Up @@ -151,14 +153,25 @@

PostgresDecimalConfig ReadDecimalConfig();

static PostgresDecimalKind NonFiniteDecimalKindFromSign(uint16_t dec_sign);

static string NonFiniteDecimalKindToString(PostgresDecimalKind kind);

template <class T, class OP = DecimalConversionInteger>
T ReadDecimal() {
PostgresDecimal<T> ReadDecimal() {
// this is wild
auto config = ReadDecimalConfig();

// we do not support non-finite numerics, need to return NULL or throw
if (config.sign == NUMERIC_NAN || config.sign == NUMERIC_PINF || config.sign == NUMERIC_NINF) {
PostgresDecimalKind kind = NonFiniteDecimalKindFromSign(config.sign);
return PostgresDecimal<T>(kind);
}

auto scale_POWER = OP::GetPowerOfTen(config.scale);

if (config.ndigits == 0) {
return 0;
return PostgresDecimal<T>(static_cast<T>(0));
}
T integral_part = 0, fractional_part = 0;

Expand Down Expand Up @@ -210,7 +223,8 @@

// finally
auto base_res = OP::Finalize(config, integral_part + fractional_part);
return (config.is_negative ? -base_res : base_res);
auto val = (config.is_negative ? -base_res : base_res);
return PostgresDecimal<T>(val);
}

void ReadGeometry(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset);
Expand All @@ -219,6 +233,8 @@
uint32_t current_count, uint32_t dimensions[], uint32_t ndim);

void ReadValue(const LogicalType &type, const PostgresType &postgres_type, Vector &out_vec, idx_t output_offset);

bool CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind);
};

} // namespace duckdb
15 changes: 15 additions & 0 deletions src/include/postgres_conversion.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ struct PostgresDecimalConfig {
uint16_t ndigits;
int16_t weight;
bool is_negative;
uint16_t sign;
};

enum class PostgresDecimalKind { ORDINARY, NOT_A_NUMBER, POSITIVE_INFINITY, NEGATIVE_INFINITY };

template <typename T>
struct PostgresDecimal {
T value;
PostgresDecimalKind kind;

PostgresDecimal(T val) : value(val), kind(PostgresDecimalKind::ORDINARY) {
}

PostgresDecimal(PostgresDecimalKind kind_p) : value(0), kind(kind_p) {
}
};

struct PostgresConversion {
Expand Down
1 change: 1 addition & 0 deletions src/include/postgres_scanner.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ struct PostgresBindData : public dbconnector::BindData {
//! DML without RETURNING). InitGlobalState executes it and returns a single-row Success result.
bool command_only = false;
idx_t max_threads = 1;
PostgresTypeConfig type_config;

dbconnector::optimizer::OrderByAndLimitBindData order_by_and_limit_bind_data;
dbconnector::optimizer::AggregateBindData aggregate_bind_data;
Expand Down
1 change: 1 addition & 0 deletions src/include/postgres_utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ struct PostgresTypeData {
struct PostgresTypeConfig {
bool array_as_varchar = false;
bool numeric_as_varchar = false;
bool numeric_nan_as_null = true;

static PostgresTypeConfig FromContext(optional_ptr<ClientContext> context);
};
Expand Down
3 changes: 2 additions & 1 deletion src/postgres_binary_copy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,9 @@ static unique_ptr<GlobalTableFunctionState> PostgresBinaryReadInitGlobal(ClientC
TableFunctionInitInput &input) {
auto &bind_data = input.bind_data->Cast<PostgresBinaryReadBindData>();
auto result = make_uniq<PostgresBinaryReadGlobalState>();
PostgresTypeConfig type_config = PostgresTypeConfig::FromContext(context);
result->reader = make_uniq<PostgresBinaryFileReader>(context, bind_data.file_path, bind_data.types,
bind_data.postgres_types, bind_data.buffer_size);
bind_data.postgres_types, type_config, bind_data.buffer_size);
return std::move(result);
}

Expand Down
8 changes: 4 additions & 4 deletions src/postgres_binary_file_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ static vector<column_t> MakeSequentialColumnIds(idx_t count) {

PostgresBinaryFileReader::PostgresBinaryFileReader(ClientContext &context, const string &file_path,
vector<LogicalType> types_p, vector<PostgresType> postgres_types_p,
idx_t buffer_size_p)
: column_ids(MakeSequentialColumnIds(types_p.size())), parser(std::move(types_p), std::move(postgres_types_p)),
buffer_size(buffer_size_p), file_offset(0), leftover(0), leftover_offset(0), finished(false),
header_scanned(false) {
PostgresTypeConfig type_config, idx_t buffer_size_p)
: column_ids(MakeSequentialColumnIds(types_p.size())),
parser(std::move(types_p), std::move(postgres_types_p), type_config), buffer_size(buffer_size_p), file_offset(0),
leftover(0), leftover_offset(0), finished(false), header_scanned(false) {
auto &fs = FileSystem::GetFileSystem(context);
file_handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ);
file_size = file_handle->GetFileSize();
Expand Down
97 changes: 85 additions & 12 deletions src/postgres_binary_parser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@

namespace duckdb {

PostgresBinaryParser::PostgresBinaryParser(vector<LogicalType> types_p, vector<PostgresType> postgres_types_p)
: types(std::move(types_p)), postgres_types(std::move(postgres_types_p)) {
PostgresBinaryParser::PostgresBinaryParser(vector<LogicalType> types_p, vector<PostgresType> postgres_types_p,
PostgresTypeConfig type_config_p)
: types(std::move(types_p)), postgres_types(std::move(postgres_types_p)), type_config(type_config_p) {
}

void PostgresBinaryParser::SetBuffer(data_ptr_t buf, idx_t len) {
Expand Down Expand Up @@ -81,6 +82,7 @@ PostgresDecimalConfig PostgresBinaryParser::ReadDecimalConfig() {
sign == NUMERIC_NEG)) {
throw NotImplementedException("Postgres numeric NA/Inf");
}
config.sign = sign;
config.is_negative = sign == NUMERIC_NEG;
config.scale = ReadInteger<uint16_t>();

Expand Down Expand Up @@ -188,7 +190,25 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType
break;
case LogicalTypeId::DOUBLE: {
if (postgres_type.info == PostgresTypeAnnotation::NUMERIC_AS_DOUBLE) {
FlatVector::GetDataMutable<double>(out_vec)[output_offset] = ReadDecimal<double, DecimalConversionDouble>();
PostgresDecimal<double> dec = ReadDecimal<double, DecimalConversionDouble>();
double double_value = 0;
switch (dec.kind) {
case PostgresDecimalKind::ORDINARY:
double_value = dec.value;
break;
case PostgresDecimalKind::NOT_A_NUMBER:
double_value = std::numeric_limits<double>::quiet_NaN();
break;
case PostgresDecimalKind::POSITIVE_INFINITY:
double_value = std::numeric_limits<double>::infinity();
break;
case PostgresDecimalKind::NEGATIVE_INFINITY:
double_value = -std::numeric_limits<double>::infinity();
break;
default:
throw InvalidInputException("Unsupported decimal kind");
}
FlatVector::GetDataMutable<double>(out_vec)[output_offset] = double_value;
break;
}
D_ASSERT(value_len == sizeof(double));
Expand Down Expand Up @@ -235,19 +255,34 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType
throw InvalidInputException("Need at least 8 bytes to read a Postgres decimal. Got %d", value_len);
}
switch (type.InternalType()) {
case PhysicalType::INT16:
FlatVector::GetDataMutable<int16_t>(out_vec)[output_offset] = ReadDecimal<int16_t>();
case PhysicalType::INT16: {
PostgresDecimal<int16_t> dec = ReadDecimal<int16_t>();
if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) {
FlatVector::GetDataMutable<int16_t>(out_vec)[output_offset] = dec.value;
}
break;
case PhysicalType::INT32:
FlatVector::GetDataMutable<int32_t>(out_vec)[output_offset] = ReadDecimal<int32_t>();
}
case PhysicalType::INT32: {
PostgresDecimal<int32_t> dec = ReadDecimal<int32_t>();
if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) {
FlatVector::GetDataMutable<int32_t>(out_vec)[output_offset] = dec.value;
}
break;
case PhysicalType::INT64:
FlatVector::GetDataMutable<int64_t>(out_vec)[output_offset] = ReadDecimal<int64_t>();
}
case PhysicalType::INT64: {
PostgresDecimal<int64_t> dec = ReadDecimal<int64_t>();
if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) {
FlatVector::GetDataMutable<int64_t>(out_vec)[output_offset] = dec.value;
}
break;
case PhysicalType::INT128:
FlatVector::GetDataMutable<hugeint_t>(out_vec)[output_offset] =
ReadDecimal<hugeint_t, DecimalConversionHugeint>();
}
case PhysicalType::INT128: {
PostgresDecimal<hugeint_t> dec = ReadDecimal<hugeint_t, DecimalConversionHugeint>();
if (!CheckDecimalKindSetNull(out_vec, output_offset, dec.kind)) {
FlatVector::GetDataMutable<hugeint_t>(out_vec)[output_offset] = dec.value;
}
break;
}
default:
throw InvalidInputException("Unsupported decimal storage type");
}
Expand Down Expand Up @@ -387,4 +422,42 @@ void PostgresBinaryParser::ReadValue(const LogicalType &type, const PostgresType
}
}

PostgresDecimalKind PostgresBinaryParser::NonFiniteDecimalKindFromSign(uint16_t dec_sign) {
if (dec_sign == NUMERIC_NAN) {
return PostgresDecimalKind::NOT_A_NUMBER;
}
if (dec_sign == NUMERIC_PINF) {
return PostgresDecimalKind::POSITIVE_INFINITY;
}
if (dec_sign == NUMERIC_NINF) {
return PostgresDecimalKind::NEGATIVE_INFINITY;
}
throw InvalidInputException("Unsupported unbound NUMERIC sign: %u", static_cast<unsigned int>(dec_sign));
}

string PostgresBinaryParser::NonFiniteDecimalKindToString(PostgresDecimalKind kind) {
switch (kind) {
case PostgresDecimalKind::NOT_A_NUMBER:
return "NaN";
case PostgresDecimalKind::POSITIVE_INFINITY:
return "Infinity";
case PostgresDecimalKind::NEGATIVE_INFINITY:
return "-Infinity";
default:
throw InvalidInputException("Unsupported unbound NUMERIC kind");
}
}

bool PostgresBinaryParser::CheckDecimalKindSetNull(Vector &out_vec, idx_t output_offset, PostgresDecimalKind dec_kind) {
if (dec_kind == PostgresDecimalKind::ORDINARY) {
return false;
}
if (!type_config.numeric_nan_as_null) {
string kind_str = NonFiniteDecimalKindToString(dec_kind);
throw InvalidInputException("Unsupported NUMERIC value: %s", kind_str);
}
FlatVector::SetNull(out_vec, output_offset, true);
return true;
}

} // namespace duckdb
3 changes: 2 additions & 1 deletion src/postgres_binary_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ namespace duckdb {

PostgresBinaryReader::PostgresBinaryReader(PostgresConnection &con_p, const vector<column_t> &column_ids,
const PostgresBindData &bind_data)
: PostgresResultReader(con_p, column_ids, bind_data), parser(bind_data.types, bind_data.postgres_types) {
: PostgresResultReader(con_p, column_ids, bind_data),
parser(bind_data.types, bind_data.postgres_types, bind_data.type_config) {
}

PostgresBinaryReader::~PostgresBinaryReader() {
Expand Down
3 changes: 3 additions & 0 deletions src/postgres_extension.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,9 @@ static void LoadInternal(ExtensionLoader &loader) {
"pg_numeric_as_varchar",
"Read Postgres numerics without precision and scale or precision > 38 as varchar instead of double",
LogicalType::BOOLEAN, Value::BOOLEAN(false), PostgresClearCacheFunction::ClearCacheOnSetting);
config.AddExtensionOption(
"pg_numeric_nan_as_null", "Read Postgres numeric NaN value as NULL instead of throwing an error",
LogicalType::BOOLEAN, Value::BOOLEAN(true), PostgresClearCacheFunction::ClearCacheOnSetting);
config.AddExtensionOption(
"pg_connection_cache",
"Whether or not to use the connection pooling."
Expand Down
1 change: 1 addition & 0 deletions src/postgres_query.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ static unique_ptr<FunctionData> PGQueryBind(ClientContext &context, TableFunctio
}

// set up the bind data
result->type_config = type_config;
result->SetCatalog(pg_catalog);
result->dsn = con.GetDSN();
result->types = return_types;
Expand Down
3 changes: 3 additions & 0 deletions src/postgres_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ PostgresTypeConfig PostgresTypeConfig::FromContext(optional_ptr<ClientContext> c
if (context->TryGetCurrentSetting("pg_numeric_as_varchar", setting)) {
result.numeric_as_varchar = BooleanValue::Get(setting);
}
if (context->TryGetCurrentSetting("pg_numeric_nan_as_null", setting)) {
result.numeric_nan_as_null = BooleanValue::Get(setting);
}
return result;
}

Expand Down
1 change: 1 addition & 0 deletions src/storage/postgres_table_entry.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ TableFunction PostgresTableEntry::GetScanFunction(ClientContext &context, unique
result->names = postgres_names;
result->postgres_types = postgres_types;
result->read_only = transaction.IsReadOnly();
result->type_config = PostgresTypeConfig::FromContext(context);
PostgresScanFunction::PrepareBind(pg_catalog.GetPostgresVersion(), context, *result,
approx_num_pages.load(std::memory_order_acquire));

Expand Down
Loading
Loading