using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Npgsql.BackendMessages;
using Npgsql.Internal;
using Npgsql.Logging;
using Npgsql.PostgresTypes;
using Npgsql.Util;
using static Npgsql.Util.Statics;
// ReSharper disable StringLiteralTypo
// ReSharper disable CommentTypo
namespace Npgsql;
///
/// The default implementation of , for standard PostgreSQL databases..
///
class PostgresDatabaseInfoFactory : INpgsqlDatabaseInfoFactory
{
///
public async Task Load(NpgsqlConnector conn, NpgsqlTimeout timeout, bool async)
{
var db = new PostgresDatabaseInfo(conn);
await db.LoadPostgresInfo(conn, timeout, async);
Debug.Assert(db.LongVersion != null);
return db;
}
}
///
/// The default implementation of NpgsqlDatabase, for standard PostgreSQL databases.
///
class PostgresDatabaseInfo : NpgsqlDatabaseInfo
{
///
/// The Npgsql logger instance.
///
static readonly NpgsqlLogger Log = NpgsqlLogManager.CreateLogger(nameof(PostgresDatabaseInfo));
///
/// The PostgreSQL types detected in the database.
///
List? _types;
///
protected override IEnumerable GetTypes() => _types ?? Enumerable.Empty();
///
/// The PostgreSQL version string as returned by the version() function. Populated during loading.
///
public string LongVersion { get; set; } = default!;
///
/// True if the backend is Amazon Redshift; otherwise, false.
///
public bool IsRedshift { get; private set; }
///
public override bool SupportsUnlisten => Version.IsGreaterOrEqual(6, 4) && !IsRedshift;
///
/// True if the 'pg_enum' table includes the 'enumsortorder' column; otherwise, false.
///
public virtual bool HasEnumSortOrder => Version.IsGreaterOrEqual(9, 1);
///
/// True if the 'pg_type' table includes the 'typcategory' column; otherwise, false.
///
///
/// pg_type.typcategory is added after 8.4.
/// see: https://www.postgresql.org/docs/8.4/static/catalog-pg-type.html#CATALOG-TYPCATEGORY-TABLE
///
public virtual bool HasTypeCategory => Version.IsGreaterOrEqual(8, 4);
internal PostgresDatabaseInfo(NpgsqlConnector conn)
: base(conn.Host!, conn.Port, conn.Database!, conn.PostgresParameters["server_version"])
{
}
///
/// Loads database information from the PostgreSQL database specified by .
///
/// The database connection.
/// The timeout while loading types from the backend.
/// True to load types asynchronously.
///
/// A task representing the asynchronous operation.
///
internal async Task LoadPostgresInfo(NpgsqlConnector conn, NpgsqlTimeout timeout, bool async)
{
HasIntegerDateTimes =
conn.PostgresParameters.TryGetValue("integer_datetimes", out var intDateTimes) &&
intDateTimes == "on";
IsRedshift = conn.Settings.ServerCompatibilityMode == ServerCompatibilityMode.Redshift;
_types = await LoadBackendTypes(conn, timeout, async);
}
///
/// Generates a raw SQL query string to select type information.
///
///
/// Select all types (base, array which is also base, enum, range, composite).
/// Note that arrays are distinguished from primitive types through them having typreceive=array_recv.
/// Order by primitives first, container later.
/// For arrays and ranges, join in the element OID and type (to filter out arrays of unhandled
/// types).
///
static string GenerateLoadTypesQuery(bool withRange, bool withMultirange, bool loadTableComposites)
=> $@"
SELECT ns.nspname, t.oid, t.typname, t.typtype, t.typnotnull, t.elemtypoid
FROM (
-- Arrays have typtype=b - this subquery identifies them by their typreceive and converts their typtype to a
-- We first do this for the type (innerest-most subquery), and then for its element type
-- This also returns the array element, range subtype and domain base type as elemtypoid
SELECT
typ.oid, typ.typnamespace, typ.typname, typ.typtype, typ.typrelid, typ.typnotnull, typ.relkind,
elemtyp.oid AS elemtypoid, elemtyp.typname AS elemtypname, elemcls.relkind AS elemrelkind,
CASE WHEN elemproc.proname='array_recv' THEN 'a' ELSE elemtyp.typtype END AS elemtyptype
FROM (
SELECT typ.oid, typnamespace, typname, typrelid, typnotnull, relkind, typelem AS elemoid,
CASE WHEN proc.proname='array_recv' THEN 'a' ELSE typ.typtype END AS typtype,
CASE
WHEN proc.proname='array_recv' THEN typ.typelem
{(withRange ? "WHEN typ.typtype='r' THEN rngsubtype" : "")}
{(withMultirange ? "WHEN typ.typtype='m' THEN (SELECT rngtypid FROM pg_range WHERE rngmultitypid = typ.oid)" : "")}
WHEN typ.typtype='d' THEN typ.typbasetype
END AS elemtypoid
FROM pg_type AS typ
LEFT JOIN pg_class AS cls ON (cls.oid = typ.typrelid)
LEFT JOIN pg_proc AS proc ON proc.oid = typ.typreceive
{(withRange ? "LEFT JOIN pg_range ON (pg_range.rngtypid = typ.oid)" : "")}
) AS typ
LEFT JOIN pg_type AS elemtyp ON elemtyp.oid = elemtypoid
LEFT JOIN pg_class AS elemcls ON (elemcls.oid = elemtyp.typrelid)
LEFT JOIN pg_proc AS elemproc ON elemproc.oid = elemtyp.typreceive
) AS t
JOIN pg_namespace AS ns ON (ns.oid = typnamespace)
WHERE
typtype IN ('b', 'r', 'm', 'e', 'd') OR -- Base, range, multirange, enum, domain
(typtype = 'c' AND {(loadTableComposites ? "ns.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')" : "relkind='c'")}) OR -- User-defined free-standing composites (not table composites) by default
(typtype = 'p' AND typname IN ('record', 'void')) OR -- Some special supported pseudo-types
(typtype = 'a' AND ( -- Array of...
elemtyptype IN ('b', 'r', 'm', 'e', 'd') OR -- Array of base, range, multirange, enum, domain
(elemtyptype = 'p' AND elemtypname IN ('record', 'void')) OR -- Arrays of special supported pseudo-types
(elemtyptype = 'c' AND {(loadTableComposites ? "ns.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')" : "elemrelkind='c'")}) -- Array of user-defined free-standing composites (not table composites) by default
))
ORDER BY CASE
WHEN typtype IN ('b', 'e', 'p') THEN 0 -- First base types, enums, pseudo-types
WHEN typtype = 'r' THEN 1 -- Ranges after
WHEN typtype = 'm' THEN 2 -- Multiranges after
WHEN typtype = 'c' THEN 3 -- Composites after
WHEN typtype = 'd' AND elemtyptype <> 'a' THEN 4 -- Domains over non-arrays after
WHEN typtype = 'a' THEN 5 -- Arrays after
WHEN typtype = 'd' AND elemtyptype = 'a' THEN 6 -- Domains over arrays last
END;";
static string GenerateLoadCompositeTypesQuery(bool loadTableComposites)
=> $@"
-- Load field definitions for (free-standing) composite types
SELECT typ.oid, att.attname, att.atttypid
FROM pg_type AS typ
JOIN pg_namespace AS ns ON (ns.oid = typ.typnamespace)
JOIN pg_class AS cls ON (cls.oid = typ.typrelid)
JOIN pg_attribute AS att ON (att.attrelid = typ.typrelid)
WHERE
(typ.typtype = 'c' AND {(loadTableComposites ? "ns.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')" : "cls.relkind='c'")}) AND
attnum > 0 AND -- Don't load system attributes
NOT attisdropped
ORDER BY typ.oid, att.attnum;";
static string GenerateLoadEnumFieldsQuery(bool withEnumSortOrder)
=> $@"
-- Load enum fields
SELECT pg_type.oid, enumlabel
FROM pg_enum
JOIN pg_type ON pg_type.oid=enumtypid
ORDER BY oid{(withEnumSortOrder ? ", enumsortorder" : "")};";
///
/// Loads type information from the backend specified by .
///
/// The database connection.
/// The timeout while loading types from the backend.
/// True to load types asynchronously.
///
/// A collection of types loaded from the backend.
///
///
/// Unknown typtype for type '{internalName}' in pg_type: {typeChar}.
internal async Task> LoadBackendTypes(NpgsqlConnector conn, NpgsqlTimeout timeout, bool async)
{
var commandTimeout = 0; // Default to infinity
if (timeout.IsSet)
commandTimeout = (int)timeout.CheckAndGetTimeLeft().TotalSeconds;
var versionQuery = "SELECT version();";
var loadTypesQuery = GenerateLoadTypesQuery(SupportsRangeTypes, SupportsMultirangeTypes, conn.Settings.LoadTableComposites);
var loadCompositeTypesQuery = GenerateLoadCompositeTypesQuery(conn.Settings.LoadTableComposites);
var loadEnumFieldsQuery = SupportsEnumTypes
? GenerateLoadEnumFieldsQuery(HasEnumSortOrder)
: string.Empty;
timeout.CheckAndApply(conn);
// The Lexer (https://github.com/postgres/postgres/blob/master/src/backend/replication/repl_scanner.l)
// and Parser (https://github.com/postgres/postgres/blob/master/src/backend/replication/repl_gram.y)
// for replication connections are pretty picky and somewhat flawed.
// Currently (2022-01-22) they do not support
// - SQL batches containing multiple commands
// - The ('\r') in Windows or Mac newlines
// - Comments
// For this reason we need clean up our type loading queries for replication connections and execute
// them individually instead of batched.
// Theoretically we cold even use the extended protocol + batching for regular (non-replication)
// connections but that would branch our code even more for very little gain.
var isReplicationConnection = conn.Settings.ReplicationMode != ReplicationMode.Off;
if (isReplicationConnection)
{
await conn.WriteQuery(versionQuery, async);
await conn.WriteQuery(SanitizeForReplicationConnection(loadTypesQuery), async);
await conn.WriteQuery(SanitizeForReplicationConnection(loadCompositeTypesQuery), async);
if (SupportsEnumTypes)
await conn.WriteQuery(SanitizeForReplicationConnection(loadEnumFieldsQuery), async);
static string SanitizeForReplicationConnection(string str)
{
var sb = new StringBuilder(str.Length);
using var c = str.GetEnumerator();
while (c.MoveNext())
{
switch (c.Current)
{
case '\r':
sb.Append('\n');
// Check for a \n after the \r
// and swallow it if it exists
if (c.MoveNext())
{
if (c.Current == '-')
goto case '-';
if (c.Current != '\n')
sb.Append(c.Current);
}
break;
case '-':
// Check if there is a second dash
if (c.MoveNext())
{
if (c.Current == '\r')
{
sb.Append('-');
goto case '\r';
}
if (c.Current != '-')
{
sb.Append('-');
sb.Append(c.Current);
break;
}
// Comment mode
// Swallow everything until we find a newline
while (c.MoveNext())
{
if (c.Current == '\r')
goto case '\r';
if (c.Current == '\n')
{
sb.Append('\n');
break;
}
}
}
break;
default:
sb.Append(c.Current);
break;
}
}
return sb.ToString();
}
}
else
{
var batchQuery = new StringBuilder(
versionQuery.Length +
loadTypesQuery.Length +
loadCompositeTypesQuery.Length +
(SupportsEnumTypes
? loadEnumFieldsQuery.Length
: 0))
.AppendLine(versionQuery)
.AppendLine(loadTypesQuery)
.AppendLine(loadCompositeTypesQuery);
if (SupportsEnumTypes)
batchQuery.AppendLine(loadEnumFieldsQuery);
await conn.WriteQuery(batchQuery.ToString(), async);
}
await conn.Flush(async);
var byOID = new Dictionary();
var buf = conn.ReadBuffer;
// First read the PostgreSQL version
Expect(await conn.ReadMessage(async), conn);
// We read the message in non-sequential mode which buffers the whole message.
// There is no need to ensure data within the message boundaries
Expect(await conn.ReadMessage(async), conn);
buf.Skip(2); // Column count
LongVersion = ReadNonNullableString(buf);
Expect(await conn.ReadMessage(async), conn);
if (isReplicationConnection)
Expect(await conn.ReadMessage(async), conn);
// Then load the types
Expect(await conn.ReadMessage(async), conn);
IBackendMessage msg;
while (true)
{
msg = await conn.ReadMessage(async);
if (msg is not DataRowMessage)
break;
buf.Skip(2); // Column count
var nspname = ReadNonNullableString(buf);
var oid = uint.Parse(ReadNonNullableString(buf), NumberFormatInfo.InvariantInfo);
Debug.Assert(oid != 0);
var typname = ReadNonNullableString(buf);
var typtype = ReadNonNullableString(buf)[0];
var typnotnull = ReadNonNullableString(buf)[0] == 't';
var len = buf.ReadInt32();
var elemtypoid = len == -1 ? 0 : uint.Parse(buf.ReadString(len), NumberFormatInfo.InvariantInfo);
switch (typtype)
{
case 'b': // Normal base type
var baseType = new PostgresBaseType(nspname, typname, oid);
byOID[baseType.OID] = baseType;
continue;
case 'a': // Array
{
Debug.Assert(elemtypoid > 0);
if (!byOID.TryGetValue(elemtypoid, out var elementPostgresType))
{
Log.Trace($"Array type '{typname}' refers to unknown element with OID {elemtypoid}, skipping", conn.Id);
continue;
}
var arrayType = new PostgresArrayType(nspname, typname, oid, elementPostgresType);
byOID[arrayType.OID] = arrayType;
continue;
}
case 'r': // Range
{
Debug.Assert(elemtypoid > 0);
if (!byOID.TryGetValue(elemtypoid, out var subtypePostgresType))
{
Log.Trace($"Range type '{typname}' refers to unknown subtype with OID {elemtypoid}, skipping", conn.Id);
continue;
}
var rangeType = new PostgresRangeType(nspname, typname, oid, subtypePostgresType);
byOID[rangeType.OID] = rangeType;
continue;
}
case 'm': // Multirange
Debug.Assert(elemtypoid > 0);
if (!byOID.TryGetValue(elemtypoid, out var type))
{
Log.Trace($"Multirange type '{typname}' refers to unknown range with OID {elemtypoid}, skipping", conn.Id);
continue;
}
if (type is not PostgresRangeType rangePostgresType)
{
Log.Trace($"Multirange type '{typname}' refers to non-range type {type.Name}, skipping",
conn.Id);
continue;
}
var multirangeType = new PostgresMultirangeType(nspname, typname, oid, rangePostgresType);
byOID[multirangeType.OID] = multirangeType;
continue;
case 'e': // Enum
var enumType = new PostgresEnumType(nspname, typname, oid);
byOID[enumType.OID] = enumType;
continue;
case 'c': // Composite
var compositeType = new PostgresCompositeType(nspname, typname, oid);
byOID[compositeType.OID] = compositeType;
continue;
case 'd': // Domain
Debug.Assert(elemtypoid > 0);
if (!byOID.TryGetValue(elemtypoid, out var basePostgresType))
{
Log.Trace($"Domain type '{typname}' refers to unknown base type with OID {elemtypoid}, skipping", conn.Id);
continue;
}
var domainType = new PostgresDomainType(nspname, typname, oid, basePostgresType, typnotnull);
byOID[domainType.OID] = domainType;
continue;
case 'p': // pseudo-type (record, void)
goto case 'b'; // Hack this as a base type
default:
throw new ArgumentOutOfRangeException($"Unknown typtype for type '{typname}' in pg_type: {typtype}");
}
}
Expect(msg, conn);
if (isReplicationConnection)
Expect(await conn.ReadMessage(async), conn);
// Then load the composite type fields
Expect(await conn.ReadMessage(async), conn);
var currentOID = uint.MaxValue;
PostgresCompositeType? currentComposite = null;
var skipCurrent = false;
while (true)
{
msg = await conn.ReadMessage(async);
if (msg is not DataRowMessage)
break;
buf.Skip(2); // Column count
var oid = uint.Parse(ReadNonNullableString(buf), NumberFormatInfo.InvariantInfo);
var attname = ReadNonNullableString(buf);
var atttypid = uint.Parse(ReadNonNullableString(buf), NumberFormatInfo.InvariantInfo);
if (oid != currentOID)
{
currentOID = oid;
if (!byOID.TryGetValue(oid, out var type)) // See #2020
{
Log.Warn($"Skipping composite type with OID {oid} which was not found in pg_type");
byOID.Remove(oid);
skipCurrent = true;
continue;
}
currentComposite = type as PostgresCompositeType;
if (currentComposite == null)
{
Log.Warn($"Type {type.Name} was referenced as a composite type but is a {type.GetType()}");
byOID.Remove(oid);
skipCurrent = true;
continue;
}
skipCurrent = false;
}
if (skipCurrent)
continue;
if (!byOID.TryGetValue(atttypid, out var fieldType)) // See #2020
{
Log.Warn($"Skipping composite type {currentComposite!.DisplayName} with field {attname} with type OID {atttypid}, which could not be resolved to a PostgreSQL type.");
byOID.Remove(oid);
skipCurrent = true;
continue;
}
currentComposite!.MutableFields.Add(new PostgresCompositeType.Field(attname, fieldType));
}
Expect(msg, conn);
if (isReplicationConnection)
Expect(await conn.ReadMessage(async), conn);
if (SupportsEnumTypes)
{
// Then load the enum fields
Expect(await conn.ReadMessage(async), conn);
currentOID = uint.MaxValue;
PostgresEnumType? currentEnum = null;
skipCurrent = false;
while (true)
{
msg = await conn.ReadMessage(async);
if (msg is not DataRowMessage)
break;
buf.Skip(2); // Column count
var oid = uint.Parse(ReadNonNullableString(buf), NumberFormatInfo.InvariantInfo);
var enumlabel = ReadNonNullableString(buf);
if (oid != currentOID)
{
currentOID = oid;
if (!byOID.TryGetValue(oid, out var type)) // See #2020
{
Log.Warn($"Skipping enum type with OID {oid} which was not found in pg_type");
byOID.Remove(oid);
skipCurrent = true;
continue;
}
currentEnum = type as PostgresEnumType;
if (currentEnum == null)
{
Log.Warn($"Type {type.Name} was referenced as an enum type but is a {type.GetType()}");
byOID.Remove(oid);
skipCurrent = true;
continue;
}
skipCurrent = false;
}
if (skipCurrent)
continue;
currentEnum!.MutableLabels.Add(enumlabel);
}
Expect(msg, conn);
if (isReplicationConnection)
Expect(await conn.ReadMessage(async), conn);
}
if (!isReplicationConnection)
Expect(await conn.ReadMessage(async), conn);
return byOID.Values.ToList();
static string ReadNonNullableString(NpgsqlReadBuffer buffer)
=> buffer.ReadString(buffer.ReadInt32());
}
}