Spark Scala项目单元测试遇json4s依赖缺失问题求助

API 探路者

2026-6-22

问题描述

使用Scala 2.12、Spark 3.1.2和JDK 1.8时，遇到依赖缺失错误：

A needed class was not found. This could be due to an error in your runpath. Missing class: org/json4s/JsonAST$JValue

项目为多模块Gradle结构，试图复用common-test-utils模块中的SparkTestingFixture实现Spark单元测试复用，但陷入json4s依赖冲突。项目结构如下：

├───module_a
     build.gradle
│    src/
      main/
       scala/
        SparkTransformationLogic.scala
      tests/
        TestSparkTransformationLogic.scala
├───common-test-utils
    build.gradle
│   src/
     main/
      scala/
       SparkTestingFixture.scala
├───module_c
    build.gradle
     main/
       scala/
        FooBarLogic.scala
      tests/
        TestFooBarLogic.scala
build.gradle

各模块配置如下：

Module A build.gradle

apply plugin: 'com.company.gradle.foobar.library'

dependencies {
    implementation project(":common-test-utils")

    implementation("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion"){
        exclude group: "org.apache.logging.log4j"
    }
    implementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
    implementation ("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion"){
        exclude group: "org.apache.logging.log4j"
    }
    implementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
    implementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"

    implementation "com.company.yahoobug:jvm-config-api:$jvmConfigVersion"
    implementation "com.company.yahoobug:sparkutils-environment:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-job:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-logging:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-spark:$sparkUtilsVersion"

    testImplementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"
    testImplementation "org.slf4j:slf4j-api:1.7.36"
    testImplementation "org.json4s:json4s-jackson_2.12:3.6.7"
}

common-test-utils build.gradle

plugins {
    id 'scala'
}

apply plugin: "java"
apply plugin: "scala"

dependencies {
    implementation("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion")
    implementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
    implementation("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion")
    implementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
    implementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"
    implementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"

    implementation "org.json4s:json4s-core_2.12:3.6.7"
    implementation "org.json4s:json4s-native_2.12:3.6.7"
    implementation "org.json4s:json4s-jackson_2.12:3.6.7"
}

Root项目build.gradle

plugins {
    id 'com.foobar.gradle.repos' version '0.3.10'
    id 'com.foobar.gradle.foozy.library' version "1.0.23"
    id 'com.github.maiflai.scalatest' version '0.35'
    id 'com.foobar.gradle.checks' version '1.2.3'
}

apply from: 'versions.gradle'

allprojects {
    apply plugin: "com.github.maiflai.scalatest"
    group 'com.foobar.foozy'
}

subprojects {

    apply plugin: 'java-library'
    apply plugin: 'scala'
    apply plugin: "com.foorbar.gradle.repos"
    apply plugin: "com.github.maiflai.scalatest"

    fooBarRepositoryPlugin {
        forceGuavaVersion = false
    }

    tasks.withType(JavaCompile) {
        sourceCompatibility = "1.8"
        targetCompatibility = "1.8"
    }

    configurations {
        sparkTestImplementation
        sparkTestRuntimeOnly
    }

    dependencies {
        implementation "org.scala-lang:scala-library:$scalaVersion"
        implementation "org.scala-lang.modules:scala-java8-compat_$scalaMajorVersion:0.9.1"
        implementation "org.openjdk.jol:jol-core:0.9" // for local object size diagnostics
        implementation "commons-io:commons-io:2.4"

        implementation "org.apache.hadoop:hadoop-common:$hadoopVersion"
        implementation "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
        implementation("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion") {
            exclude group: "org.apache.logging.log4j"
        }
        implementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
        implementation "org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion"
        implementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
        implementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"

        implementation "org.xerial.snappy:snappy-java:1.1.2.1"
        implementation "com.amazonaws:aws-java-sdk-s3:1.11.333"
        implementation "com.amazonaws:aws-java-sdk-ec2:1.11.333"
        implementation "com.amazonaws:aws-java-sdk-kinesis:1.11.333"

        implementation "joda-time:joda-time:$jodaVersion"
        implementation "com.github.nscala-time:nscala-time_$scalaMajorVersion:2.14.0"

        compile "com.datadoghq:java-dogstatsd-client:$dogstatsdVersion"

        testImplementation "junit:junit:4.12"
        testImplementation "org.scalatestplus:scalatestplus-junit_$scalaMajorVersion:1.0.0-M2"
        testImplementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"
        testImplementation "org.mockito:mockito-all:$mockitoVersion"
        implementation "com.vladsch.flexmark:flexmark-all:0.62.2" 
        
        testRuntimeOnly "org.slf4j:slf4j-jdk14:1.7.30"
        testImplementation 'org.apache.logging.log4j:log4j-1.2-api:2.17.2'

    }

    buildscript {
        repositories {
            jcenter()
        }
        dependencies {
            classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0'
        }
    }

}

checks {
    checkScalaStyle = false
}

// Script to copy audit test data
task zipTipZap {
    description "Publish task"

    doLast {

        println("gathering audit data for $project.name")

        copy { //Test Results
            from ('.') {
                include '**/test-results/**/TEST-*.xml'
                exclude '**/build/audit_data' //exclude itself from scanning
            }
            into 'build/audit_data/test_results'
            includeEmptyDirs false
        }
    }
}

Spark Testing Fixture

import org.apache.spark.sql.SparkSession
import org.scalatest.{BeforeAndAfterEach, Suite}

import java.io.File
import java.nio.file.Files


trait SparkSessionFixture extends BeforeAndAfterEach { this: Suite =>
  @transient var spark: SparkSession = _

  val warehouseLocation: String = Files.createTempDirectory("spark-warehouse").toFile.getAbsolutePath
  val metastoreLocation: String = Files.createTempDirectory("spark-metastore").toFile.getAbsolutePath
  val tempHadoopDir: File = Files.createTempDirectory("hadoop_home").toFile

  System.setProperty("derby.system.home", metastoreLocation)
  System.setProperty("hadoop.home.dir", tempHadoopDir.getAbsolutePath)

  override protected def beforeEach(): Unit = {
    super.beforeEach()
    spark = SparkSession
      .builder()
      .master("local")
      .appName("spark session")
      .config("spark.ui.enabled", "false")
      .config("spark.sql.shuffle.partitions", "1")
      .config("spark.sql.warehouse.dir", warehouseLocation)
      .config("javax.jdo.option.ConnectionURL", s"jdbc:derby:;databaseName=metastore_db;create=true")
      .enableHiveSupport()
      .getOrCreate()

  }

  override protected def afterEach(): Unit = {
    if (spark != null) {
      spark.stop()
    }
    super.afterEach()
  }

}

依赖冲突原因分析

Spark内置json4s版本与手动引入版本不兼容：Spark 3.1.2默认依赖json4s 3.2.11，但项目中手动引入了3.6.7版本，导致类路径中出现版本冲突，部分类被覆盖或无法加载。
依赖传递范围错误：common-test-utils将测试相关的json4s依赖放在implementation范围，会传递给依赖它的module_a主代码依赖，而非仅在测试阶段生效，打乱了生产与测试依赖的隔离。
全局依赖配置混乱：Root项目的subprojects中全局引入了所有Spark模块到生产依赖，没有区分生产与测试场景，加剧了依赖版本冲突的概率。

解决方案与依赖隔离指导

1. 统一json4s版本并对齐Spark内置版本

Spark 3.1.2内置的json4s版本为3.2.11，直接使用该版本避免冲突：

在versions.gradle中添加统一版本变量：
```
json4sVersion = "3.2.11"
```
所有模块中替换原3.6.7版本为该统一变量。

2. 调整`common-test-utils`的依赖范围

测试工具类的依赖应仅在测试阶段生效，修改common-test-utils/build.gradle：

plugins {
    id 'scala'
}

apply plugin: "java-library"
apply plugin: "scala"

dependencies {
    // Spark核心依赖用api范围，让依赖该模块的测试能访问Spark类，但不传递多余依赖
    api("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion")
    api("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion")
    
    // 非核心Spark模块仅在测试时使用
    testImplementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
    testImplementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
    testImplementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"
    
    testImplementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"
    // json4s依赖改为testImplementation，仅测试时使用，不传递给主项目
    testImplementation "org.json4s:json4s-core_$scalaMajorVersion:$json4sVersion"
    testImplementation "org.json4s:json4s-native_$scalaMajorVersion:$json4sVersion"
    testImplementation "org.json4s:json4s-jackson_$scalaMajorVersion:$json4sVersion"
}

3. 隔离`module_a`的测试依赖

修改module_a/build.gradle，确保测试依赖仅在测试阶段生效：

apply plugin: 'com.company.gradle.foobar.library'

dependencies {
    // 仅测试时依赖common-test-utils
    testImplementation project(":common-test-utils")

    // 生产Spark依赖保持不变
    implementation("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion"){
        exclude group: "org.apache.logging.log4j"
    }
    implementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
    implementation ("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion"){
        exclude group: "org.apache.logging.log4j"
    }
    implementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
    implementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"

    implementation "com.company.yahoobug:jvm-config-api:$jvmConfigVersion"
    implementation "com.company.yahoobug:sparkutils-environment:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-job:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-logging:$sparkUtilsVersion"
    implementation "com.company.yahoobug:sparkutils-spark:$sparkUtilsVersion"

    testImplementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"
    testImplementation "org.slf4j:slf4j-api:1.7.36"
    // 使用统一的json4s版本
    testImplementation "org.json4s:json4s-jackson_$scalaMajorVersion:$json4sVersion"
}

4. 优化Root项目的全局依赖配置

在Root的subprojects中，区分生产与测试依赖，将非核心Spark模块移到测试范围：

subprojects {
    // ... 其他配置不变 ...

    dependencies {
        // 生产依赖仅保留核心必要模块
        implementation "org.scala-lang:scala-library:$scalaVersion"
        implementation "org.scala-lang.modules:scala-java8-compat_$scalaMajorVersion:0.9.1"
        implementation "org.openjdk.jol:jol-core:0.9"
        implementation "commons-io:commons-io:2.4"

        implementation "org.apache.hadoop:hadoop-common:$hadoopVersion"
        implementation "org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion"
        
        implementation("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion") {
            exclude group: "org.apache.logging.log4j"
        }
        implementation "org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion"

        // ... 其他生产依赖不变 ...

        // 测试依赖部分
        testImplementation "junit:junit:4.12"
        testImplementation "org.scalatestplus:scalatestplus-junit_$scalaMajorVersion:1.0.0-M2"
        testImplementation "org.scalatest:scalatest_$scalaMajorVersion:$scalaTestVersion"
        testImplementation "org.mockito:mockito-all:$mockitoVersion"
        
        // 非核心Spark模块仅在测试时引入
        testImplementation "org.apache.spark:spark-mllib_$scalaMajorVersion:$sparkVersion"
        testImplementation "org.apache.spark:spark-streaming_$scalaMajorVersion:$sparkVersion"
        testImplementation "org.apache.spark:spark-hive_$scalaMajorVersion:$sparkVersion"
        
        testRuntimeOnly "org.slf4j:slf4j-jdk14:1.7.30"
        testImplementation 'org.apache.logging.log4j:log4j-1.2-api:2.17.2'
    }
}

5. 验证依赖树

运行Gradle命令查看测试类路径的依赖树，确认json4s版本统一：

./gradlew module_a:dependencies --configuration testRuntimeClasspath

检查输出中org.json4s相关依赖是否仅存在3.2.11版本。

内容的提问来源于stack exchange，提问作者dataviews

Spark Scala项目单元测试遇json4s依赖缺失问题求助

Module A build.gradle #

common-test-utils build.gradle #

Root项目build.gradle #

Spark Testing Fixture #

1. 统一json4s版本并对齐Spark内置版本 #

2. 调整common-test-utils的依赖范围 #

3. 隔离module_a的测试依赖 #

4. 优化Root项目的全局依赖配置 #

5. 验证依赖树 #