0
트위터 분류기 - https://github.com/databricks/reference-apps을 실행하려고합니다. Spark를 사용하여 피드를 분석합니다. IntelliJ에 프로젝트를로드했고 외부 라이브러리에 나타나는 org.apache.spark.mllib 중 하나가 제대로 작동하지 않습니다.NoClassDefError in mllib 1.1.0
나는 그것을 실행에 java.lang.NoClassDefFoundError가 얻을. 하지만 이미 의존성이 존재합니다.은 다음과 같습니다
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/mllib/feature/HashingTF
at com.databricks.apps.twitter_classifier.Utils$.<init>(Utils.scala:12)
at com.databricks.apps.twitter_classifier.Utils$.<clinit>(Utils.scala)
at com.databricks.apps.twitter_classifier.Collect$.main(Collect.scala:26)
at com.databricks.apps.twitter_classifier.Collect.main(Collect.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at com.intellij.rt.execution.application.AppMain.main(AppMain.java:134)
Caused by: java.lang.ClassNotFoundException: org.apache.spark.mllib.feature.HashingTF
at java.net.URLClassLoader$1.run(URLClassLoader.java:372)
at java.net.URLClassLoader$1.run(URLClassLoader.java:361)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:360)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 9 more
내 build.sbt :
import AssemblyKeys._
name := "spark-twitter-lang-classifier"
version := "1.0"
scalaVersion := "2.10.4"
libraryDependencies += "org.apache.spark" %% "spark-core" % "1.1.0" % "provided"
libraryDependencies += "org.apache.spark" %% "spark-mllib" % "1.1.0" % "provided"
libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.1.0" % "provided"
libraryDependencies += "org.apache.spark" %% "spark-streaming" % "1.1.0" % "provided"
libraryDependencies += "org.apache.spark" %% "spark-streaming-twitter" % "1.1.0"
libraryDependencies += "com.google.code.gson" % "gson" % "2.3"
libraryDependencies += "org.twitter4j" % "twitter4j-core" % "3.0.3"
libraryDependencies += "commons-cli" % "commons-cli" % "1.2"
resolvers += "Akka Repository" at "http://repo.akka.io/releases/"
assemblySettings
mergeStrategy in assembly := {
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
case "log4j.properties" => MergeStrategy.discard
case m if m.toLowerCase.startsWith("meta-inf/services/") => MergeStrategy.filterDistinctLines
case "reference.conf" => MergeStrategy.concat
case _ => MergeStrategy.first
}