External Catalog

1 Hive Catalog

1.1 HDFS存储

• 拷贝hadoop集群中的core-site.xml, hdfs-site.xml的内容到StarRocks集群中配置管理中，重启组件后生效

1. 创建Hive Catalog

CREATE EXTERNAL CATALOG hive_catalog 
PROPERTIES 
(
"hive.metastore.uris"="thrift://{hms_ip:hms_port}", 
"type" = "hive");

2. 查询hive表记录

select * from  hive_catalog.db_1.table1 limit 1;

1.2 TOS存储

1. 创建Hive Catalog

CREATE EXTERNAL CATALOG hive_tos_catalog PROPERTIES (
    'type'='hive',
    'hive.metastore.uris' = 'thrift://{hms_ip:hms_port}',
    "aws.s3.access_key"="xxx",
    "aws.s3.secret_key"="xxx==",
    "aws.s3.endpoint"="tos-s3-cn-beijing.ivolces.com",
    "aws.s3.enable_ssl" = "false"
);

2. 查询hive tos表

select * from hive_tos_catalog.hive.hive_tos;

2 Hudi Catalog

2.1 HDFS存储

• 拷贝hadoop集群中的core-site.xml, hdfs-site.xml的内容到StarRocks集群中配置管理中，重启组件后生效

1. 创建Hudi Catalog

CREATE EXTERNAL CATALOG hudi_catalog PROPERTIES (
    'type'='hudi',
    'hive.metastore.uris' = 'thrift://{hms_ip:hms_port}'
);

2. 查询hudi表记录

select * from  hudi_catalog.db_1.table1 limit 1;

2.2 TOS存储

1. 创建Hudi Catalog

CREATE EXTERNAL CATALOG hudi_tos_catalog PROPERTIES (
    'type'='hudi',
    'hive.metastore.uris' = 'thrift://{hms_ip:hms_port}',
    "aws.s3.access_key"="xx",
    "aws.s3.secret_key"="xxx==",
    "aws.s3.endpoint"="tos-s3-cn-beijing.ivolces.com",
    "aws.s3.enable_ssl" = "false"
);

2. 查询hudi表记录

select * from  hudi_tos_catalog.db_1.table1 limit 1;

3 Iceberg Catalog

3.1 HDFS存储

1. 创建iceberg catalog

CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
    'type'='iceberg',
    'iceberg.catalog.type' = 'hive',
    'iceberg.catalog.hive.metastore.uris' = 'thrift://{hms_ip:hms_port}'
);

2. 查询iceberg表记录

select * from iceberg_catalog.iceberg.iceberg_hdfs;

3.2TOS存储

1. 创建iceberg catalog

CREATE EXTERNAL CATALOG iceberg_tos_catalog PROPERTIES (
    'type'='iceberg',
    'iceberg.catalog.type' = 'hive',
    'hive.metastore.uris' = 'thrift://{hms_ip:hms_port}',
    "aws.s3.access_key"="xxx",
    "aws.s3.secret_key"="xxxx==",
    "aws.s3.endpoint"="tos-s3-cn-beijing.ivolces.com",
    "aws.s3.enable_ssl" = "false"
);

2. 查询iceberg表记录

select * from iceberg_tos_catalog.iceberg.iceberg_tos;

4 Paimon Catalog

4.1 HDFS存储

1. 创建paimon catalog

CREATE EXTERNAL CATALOG paimon_catalog_hdfs
PROPERTIES
(
    "type" = "paimon",
    "paimon.catalog.type" = "hive",
    "paimon.catalog.warehouse" = "hdfs://master-1-1.emr-xxx.offline-cn-beijing.emr-volces.com:8020/sr-qa/paimon-warehouse",
    "hive.metastore.uris" = "thrift://{hms_ip:hms_port}"
);

2. 查询Paimon表记录

set catalog paimon_catalog_hdfs;
show databases;

use test_db_hdfs;
show tables;

select * from paimon_catalog_hdfs.test_db_hdfs.test_tbl_hdfs;

4.2 TOS存储

1. 创建paimon Catalog

CREATE EXTERNAL CATALOG paimon_catalog_fs
PROPERTIES
(
    "type" = "paimon",
    "paimon.catalog.type" = "filesystem",
    "paimon.catalog.warehouse" = "s3://sr-qa/xxx",
    "aws.s3.enable_ssl" = "false",
    "aws.s3.endpoint" = "tos-s3-cn-beijing.ivolces.com",
    "aws.s3.access_key" = "xxx",
    "aws.s3.secret_key" = "xxx=="
);

2. 查询Paimon表记录

set catalog paimon_catalog_fs;
show databases;

use test_db_fs;
show tables;

select * from paimon_catalog_fs.test_db_fs.test_tbl_fs;