如何通过Docker Compose连接Apache Hive与Apache Hadoop镜像并实现数据操作

API 探路者

2026-7-6

问题描述

我已经有一份docker-compose.yml文件，现在需要实现Apache Hive镜像与Apache Hadoop镜像的连接，具体需求包括：

在Hive中创建表
从API提取数据并加载到Hadoop HDFS
同时我想了解是否可以通过编写Dockerfile来配置两者的连接。以下是我当前的配置文件：

当前docker-compose.yml配置

version: '3.9'
services:
  postgres:
    image: postgres
    restart: unless-stopped
    container_name: postgres
    hostname: postgres
    environment:
      POSTGRES_DB: 'metastore_db'
      POSTGRES_USER: 'hive'
      POSTGRES_PASSWORD: 'hive'
    ports:
      - '5432:5432'
    volumes:
      - hive-db:/var/lib/postgresql

  namenode:
    image: apache/hadoop:3
    container_name: namenode
    hostname: namenode
    command: ["hdfs", "namenode"]
    ports:
      - 9870:9870
      - 8020:8020
    env_file:
      - ./hadoop.env
    environment:
        ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
    volumes:
      - ./data/namenode:/hadoop/dfs/name

  datanode:
    image: apache/hadoop:3
    container_name: datanode
    command: ["hdfs", "datanode"]
    ports:
      - 9864:9864
    env_file:
      - ./hadoop.env
    volumes:
      - ./data/datanode:/hadoop/dfs/data

  resourcemanager:
    image: apache/hadoop:3
    hostname: resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
        - 8088:8088
    env_file:
      - ./hadoop.env
    volumes:
      - ./test.sh:/opt/test.sh

  nodemanager:
    image: apache/hadoop:3
    command: ["yarn", "nodemanager"]
    env_file:
      - ./hadoop.env

  metastore:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - postgres
      - namenode
      - resourcemanager
      - datanode
    restart: unless-stopped
    container_name: metastore
    hostname: metastore
    environment:
      DB_DRIVER: postgres
      SERVICE_NAME: 'metastore'
      # HIVE_CUSTOM_CONF_DIR: ./hive_custom_conf 
      SERVICE_OPTS: '-Xmx1G 
        -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
        -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db
        -Djavax.jdo.option.ConnectionUserName=hive
        -Djavax.jdo.option.ConnectionPassword=hive'
    ports:
      - '9083:9083'
    volumes:
      - warehouse:/opt/hive/data/warehouse
      # - /opt/hive/conf:./hive_custom_conf

  hiveserver2:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - metastore
    restart: unless-stopped
    container_name: hiveserver2
    environment:
      HIVE_SERVER2_THRIFT_PORT: 10000
      SERVICE_OPTS: '-Xmx1G -Dhive.metastore.uris=thrift://metastore:9083'
      IS_RESUME: 'true'
      SERVICE_NAME: 'hiveserver2'
    ports:
      - '10000:10000'
      - '10002:10002'
    volumes:
      - warehouse:/opt/hive/data/warehouse

volumes:
  warehouse:
  hive-db:

networks:
  default:
    driver: bridge

当前hadoop.env配置

CORE-SITE.XML_fs.default.name=hdfs://namenode
CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
CORE-SITE.XML_hadoop.proxyuser.hue.hosts=*
CORE-SITE.XML_hadoop.proxyuser.hue.groups=*

HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
HDFS-SITE.XML_dfs.replication=1
HDFS-SITE.XML_dfs.namenode.datanode.registration.ip-hostname-check=false
HDFS-SITE.XML_dfs.webhdfs.enabled=true
HDFS-SITE.XML_dfs.fs.defaultFS=hdfs://namenode:9000
HDFS-SITE.XML_dfs.permissions.enabled=false

MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
MAPRED-SITE.XML_mapred.child.java.opts=-Xmx4096m
MAPRED-SITE.XML_mapreduce.map.memory.mb=4096
MAPRED-SITE.XML_mapreduce.reduce.memory.mb=8192
MAPRED-SITE.XML_mapreduce.map.java.opts=-Xmx3072m
MAPRED-SITE.XML_mapreduce.reduce.java.opts=-Xmx6144m

YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
YARN-SITE.XML_yarn.timeline-service.enabled=true
YARN-SITE.XML_yarn.mapreduce.map.output.compress=true
YARN-SITE.XML_yarn.mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec
YARN-SITE.XML_yarn.nodemanager.resource.memory-mb=16384
YARN-SITE.XML_yarn.nodemanager.resource.cpu-vcores=8
YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=98.5
YARN-SITE.XML_yarn.nodemanager.remote-app-log-dir=/app-logs

CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false

解决方案

一、当前配置的连接状态

你的现有docker-compose.yml已经完成了Hive与Hadoop的基础连接：

Hive Metastore通过PostgreSQL存储元数据，且依赖Hadoop核心服务（namenode、datanode等）启动
HiveServer2通过Thrift协议连接Metastore，而Metastore可通过默认Docker网络访问HDFS（Hadoop的core-site.xml已配置fs.defaultFS=hdfs://namenode:9000）
共享的warehouse卷将Hive仓库目录与HDFS对应路径绑定，Hive创建的表数据会自动存储到HDFS中

二、在Hive中创建表并加载API数据到HDFS

1. 连接HiveServer2

执行以下命令进入HiveServer2容器并使用beeline连接：

docker exec -it hiveserver2 beeline -u jdbc:hive2://localhost:10000

2. 创建Hive表

示例创建内部表（数据存储在Hive默认仓库路径）：

CREATE TABLE api_data (
    id INT,
    name STRING,
    value DOUBLE
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
STORED AS TEXTFILE;

3. 从API提取数据并加载到HDFS

方式一：容器内脚本拉取上传

在Hadoop namenode容器内执行：

# 安装curl工具
apt update && apt install -y curl
# 拉取API数据到本地
curl https://your-api-endpoint/data > /tmp/api_data.csv
# 上传到HDFS对应表路径
hdfs dfs -put /tmp/api_data.csv /user/hive/warehouse/api_data

回到beeline中加载数据：

LOAD DATA INPATH '/user/hive/warehouse/api_data/api_data.csv' INTO TABLE api_data;

方式二：自定义Python脚本（推荐）

编写脚本拉取API数据并直接上传到HDFS：

import requests
from hdfs3 import HDFileSystem

# 拉取API数据
response = requests.get("https://your-api-endpoint/data")
data = response.json()

# 转换为CSV格式
csv_content = "id,name,value\n"
for item in data:
    csv_content += f"{item['id']},{item['name']},{item['value']}\n"

# 上传到HDFS
hdfs = HDFileSystem(host='namenode', port=9000)
with hdfs.open('/user/hive/warehouse/api_data/api_data.csv', 'w') as f:
    f.write(csv_content.encode())

三、通过Dockerfile配置Hive与Hadoop连接

完全可以通过自定义Dockerfile预先配置Hive的连接参数，避免在docker-compose中重复设置环境变量。

自定义Hive镜像Dockerfile

FROM apache/hive:4.0.0-alpha-2

# 复制自定义Hive配置文件
COPY hive-site.xml /opt/hive/conf/hive-site.xml

# 配置环境变量
ENV HIVE_METASTORE_URIS=thrift://metastore:9083
ENV HADOOP_HOME=/opt/hadoop
ENV PATH=$PATH:$HADOOP_HOME/bin

# 安装依赖工具
RUN apt update && apt install -y curl python3 python3-pip
RUN pip3 install requests hdfs3

配套hive-site.xml关键配置

<configuration>
    <!-- 连接Metastore -->
    <property>
        <name>hive.metastore.uris</name>
        <value>thrift://metastore:9083</value>
    </property>
    <!-- HDFS地址配置 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://namenode:9000</value>
    </property>
    <!-- 元数据库连接 -->
    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>org.postgresql.Driver</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:postgresql://postgres:5432/metastore_db</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>hive</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>hive</value>
    </property>
</configuration>

修改docker-compose.yml使用自定义镜像

将metastore和hiveserver2的image字段替换为构建指令：

metastore:
    build: ./hive-custom  # 指向Dockerfile所在目录
    depends_on:
      - postgres
      - namenode
      - resourcemanager
      - datanode
    restart: unless-stopped
    container_name: metastore
    hostname: metastore
    ports:
      - '9083:9083'
    volumes:
      - warehouse:/opt/hive/data/warehouse

hiveserver2:
    build: ./hive-custom
    depends_on:
      - metastore
    restart: unless-stopped
    container_name: hiveserver2
    environment:
      HIVE_SERVER2_THRIFT_PORT: 10000
      IS_RESUME: 'true'
      SERVICE_NAME: 'hiveserver2'
    ports:
      - '10000:10000'
      - '10002:10002'
    volumes:
      - warehouse:/opt/hive/data/warehouse

内容的提问来源于stack exchange，提问作者blue

如何通过Docker Compose连接Apache Hive与Apache Hadoop镜像并实现数据操作

当前docker-compose.yml配置 #

当前hadoop.env配置 #

一、当前配置的连接状态 #

二、在Hive中创建表并加载API数据到HDFS #

1. 连接HiveServer2 #

2. 创建Hive表 #

3. 从API提取数据并加载到HDFS #