使用Python与DLP检测结果实现GCP Data Catalog列标签打标
基于GCP DLP检测结果同步Data Catalog列标签Python实现方案
前置依赖
- 安装对应GCP SDK包:
pip install google-cloud-dlp google-cloud-datacatalog google-cloud-bigquery
- 服务账号需配置以下权限:
- DLP Job User、DLP Job Reader
- Data Catalog Tag Editor、Data Catalog Entry Viewer
- BigQuery Data Viewer(如果检测源为BigQuery表)
核心实现代码
第一步:运行DLP列级检测并提取结果
from google.cloud import dlp_v2 def run_dlp_inspect(project_id, bq_dataset_id, bq_table_id, info_types, min_confidence="POSSIBLE"): dlp_client = dlp_v2.DlpServiceClient() parent = f"projects/{project_id}/locations/global" # 配置检测数据源 bq_table = dlp_v2.BigQueryTable( project_id=project_id, dataset_id=bq_dataset_id, table_id=bq_table_id ) storage_config = dlp_v2.StorageConfig( big_query_options=dlp_v2.BigQueryOptions( table_reference=bq_table, # 开启列级检测 sample_method=dlp_v2.BigQueryOptions.SampleMethod.TOP, rows_limit=10000 ) ) # 配置检测规则 inspect_config = dlp_v2.InspectConfig( info_types=[{"name": it} for it in info_types], min_likelihood=min_confidence, include_quote=True ) # 提交DLP作业 operation = dlp_client.create_dlp_job( request={ "parent": parent, "inspect_job": {"inspect_config": inspect_config, "storage_config": storage_config} } ) # 等待作业完成 job = operation.result() # 解析列级检测结果 column_results = {} for result in job.inspect_details.result.findings: col_name = result.location.content_location.record_location.field_id.name info_type = result.info_type.name if col_name not in column_results: column_results[col_name] = set() column_results[col_name].add(info_type) return {k: list(v) for k, v in column_results.items()}
第二步:将检测结果写入Data Catalog列标签
from google.cloud import datacatalog_v1 def tag_columns_with_dlp_results(project_id, bq_dataset_id, bq_table_id, column_results, tag_template_id): dc_client = datacatalog_v1.DataCatalogClient() # 查找对应BigQuery表的Data Catalog条目 table_resource_name = f"//bigquery.googleapis.com/projects/{project_id}/datasets/{bq_dataset_id}/tables/{bq_table_id}" lookup_request = datacatalog_v1.LookupEntryRequest(linked_resource=table_resource_name) table_entry = dc_client.lookup_entry(request=lookup_request) # 遍历表的列条目打标签 for column in table_entry.schema.columns: col_name = column.column if col_name not in column_results: continue # 构造标签 tag = datacatalog_v1.Tag() tag.template = tag_template_id tag.fields["detected_info_types"] = datacatalog_v1.TagField( string_value=",".join(column_results[col_name]) ) tag.column = col_name # 创建标签 dc_client.create_tag(parent=table_entry.name, tag=tag)
使用示例
if __name__ == "__main__": # 替换为你的实际参数 PROJECT_ID = "your-gcp-project-id" BQ_DATASET = "your-dataset-id" BQ_TABLE = "your-table-id" # 要检测的infoType列表,可根据需求调整 DETECT_INFO_TYPES = ["EMAIL_ADDRESS", "PHONE_NUMBER", "US_SOCIAL_SECURITY_NUMBER"] # 替换为你提前创建的Data Catalog标签模板ID TAG_TEMPLATE = f"projects/{PROJECT_ID}/locations/global/tagTemplates/dlp_detected_types" # 运行检测 col_dlp_results = run_dlp_inspect(PROJECT_ID, BQ_DATASET, BQ_TABLE, DETECT_INFO_TYPES) # 同步标签 tag_columns_with_dlp_results(PROJECT_ID, BQ_DATASET, BQ_TABLE, col_dlp_results, TAG_TEMPLATE)
注意事项
- 可根据业务需求调整
min_confidence参数过滤低置信度检测结果,减少误标 - 标签模板需提前创建,模板中需包含
detected_info_types的字符串类型字段 - 大表检测场景下可调整
rows_limit参数控制抽样检测的数据量,降低检测成本
内容的提问来源于stack exchange,提问作者N Mohtar
相关产品推荐
相关产品推荐

