如何在.NET Core中配置Apache Spark SQL连接？找不到对应构建类

我想在代码的(ConnectionStringofSpark)处编写Spark对应的连接构建函数，但未能找到。我希望它能像PostgreSQL或SQL Server那样有对应的实现，请问是否存在Spark自身的Spark SQL数据库连接函数（而非连接其他数据源的函数）？

以下是我编写的代码：

public string ConnectionString
{
    get
    {
        var dbInfo = DBInfo as SparkDBInfo;
        var builder = new (ConnectionStringofSpark)
        {
            Database = dbInfo.Database,
            Host = dbInfo.Server
        };
        if (dbInfo.Port > 0)
        {
            builder.Port = dbInfo.Port;
        }
        else
        {
            builder.Port = xxxx;
        }
        builder.IntegratedSecurity = dbInfo.SSPI;

        if (!dbInfo.SSPI)
        {
            builder.Username = dbInfo.Username;
            builder.Password = dbInfo.Password;
        }

        return builder.ConnectionString;
    }
}

核心结论 #

.NET生态中没有官方提供的、类似SqlConnectionStringBuilder或NpgsqlConnectionStringBuilder的Spark SQL专属连接字符串构建器。这是因为Spark SQL的交互模式和传统关系型数据库差异较大，主要分为两种主流方式：直接通过Spark .NET SDK操作，或通过JDBC连接Spark Thrift Server。

解决方案1：使用Spark .NET SDK直接交互（推荐） #

Spark .NET SDK通过SparkSession建立与Spark集群的连接，而非传统的连接字符串模式。你可以封装一个自定义的配置类来模拟连接字符串构建的逻辑：

// 自定义Spark连接配置构建类
public class SparkConnectionConfigBuilder
{
    public string Host { get; set; }
    public int Port { get; set; } = 7077; // Spark Master默认端口
    public string Database { get; set; }
    public bool UseIntegratedSecurity { get; set; }
    public string Username { get; set; }
    public string Password { get; set; }

    // 生成Spark Master连接URL
    public string GetSparkMasterUrl()
    {
        return $"spark://{Host}:{Port}";
    }

    // 创建并返回SparkSession
    public SparkSession BuildSparkSession(string appName = "Spark SQL App")
    {
        var sessionBuilder = SparkSession.Builder()
            .AppName(appName)
            .Master(GetSparkMasterUrl());

        // 如果需要指定默认数据库
        if (!string.IsNullOrEmpty(Database))
        {
            sessionBuilder.Config("spark.sql.catalogImplementation", "hive");
            sessionBuilder.Config("spark.sql.defaultCatalog", Database);
        }

        // 集成安全配置（如Kerberos）
        if (UseIntegratedSecurity)
        {
            sessionBuilder.Config("spark.security.kerberos.enabled", "true");
            // 补充其他Kerberos相关配置
        }
        else if (!string.IsNullOrEmpty(Username) && !string.IsNullOrEmpty(Password))
        {
            sessionBuilder.Config("spark.authenticate", "true");
            sessionBuilder.Config("spark.authenticate.secret", Password);
            sessionBuilder.Config("spark.ui.view.acls", Username);
        }

        return sessionBuilder.GetOrCreate();
    }
}

适配你原有代码的使用示例：

public SparkSession GetSparkSession()
{
    var dbInfo = DBInfo as SparkDBInfo;
    var builder = new SparkConnectionConfigBuilder
    {
        Database = dbInfo.Database,
        Host = dbInfo.Server,
        Port = dbInfo.Port > 0 ? dbInfo.Port : 7077,
        UseIntegratedSecurity = dbInfo.SSPI
    };

    if (!dbInfo.SSPI)
    {
        builder.Username = dbInfo.Username;
        builder.Password = dbInfo.Password;
    }

    return builder.BuildSparkSession();
}

// 使用SparkSession执行SQL
var sparkSession = GetSparkSession();
var resultDf = sparkSession.Sql("SELECT * FROM your_table");

解决方案2：通过JDBC连接Spark Thrift Server #

如果需要用ADO.NET风格的连接字符串（比如兼容现有数据库访问框架），可以连接Spark Thrift Server（HiveServer2兼容），此时可以用通用的DbConnectionStringBuilder或自定义构建类生成JDBC连接字符串：

public class SparkJdbcConnectionStringBuilder
{
    public string Host { get; set; }
    public int Port { get; set; } = 10000; // Thrift Server默认端口
    public string Database { get; set; }
    public bool IntegratedSecurity { get; set; }
    public string Username { get; set; }
    public string Password { get; set; }

    public string BuildConnectionString()
    {
        var jdbcBase = $"jdbc:hive2://{Host}:{Port}/{string.IsNullOrEmpty(Database) ? "" : Database}";
        var authParams = new List<string>();

        if (IntegratedSecurity)
        {
            authParams.Add("auth=kerberos");
        }
        else
        {
            if (!string.IsNullOrEmpty(Username)) authParams.Add($"user={Username}");
            if (!string.IsNullOrEmpty(Password)) authParams.Add($"password={Password}");
        }

        return authParams.Any() ? $"{jdbcBase};{string.Join(";", authParams)}" : jdbcBase;
    }
}

适配你原有代码的示例：

public string ConnectionString
{
    get
    {
        var dbInfo = DBInfo as SparkDBInfo;
        var builder = new SparkJdbcConnectionStringBuilder
        {
            Database = dbInfo.Database,
            Host = dbInfo.Server,
            Port = dbInfo.Port > 0 ? dbInfo.Port : 10000,
            IntegratedSecurity = dbInfo.SSPI
        };

        if (!dbInfo.SSPI)
        {
            builder.Username = dbInfo.Username;
            builder.Password = dbInfo.Password;
        }

        return builder.BuildConnectionString();
    }
}

// 使用JDBC连接示例
using var connection = new OleDbConnection(ConnectionString);
connection.Open();
// 执行SQL操作...

内容的提问来源于stack exchange，提问作者Ada