couchbase로 Apache Spark 연결하기

Question

스파크 애플리케이션을 Couchbase에 연결하려고합니다. 이를 위해 다음 코드를 적용합니다.

double[] val=new double[3]; 
SparkContext sc = new SparkContext(new SparkConf().setAppName("sql").setMaster("local").set("com.couchbase.nodes", "url").set("com.couchbase.client.bucket","password")); 
     SQLContext sql = new SQLContext(sc); 
    JsonObject content = JsonObject.create().put("mean", val[0]).put("median", val[1]).put("standardDeviation", 
        val[2]); 
      JsonDocument doc=JsonDocument.create("docId", content); 
bucket.upsert(doc); 

하지만

Exception in thread "main" java.lang.NoClassDefFoundError: com/couchbase/client/java/document/json/JsonObject 
     at com.cloudera.sparkwordcount.JavaWordCount.main(JavaWordCount.java:74) 
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) 
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
     at java.lang.reflect.Method.invoke(Method.java:606) 
     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731) 
     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) 
     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) 
     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) 
     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 
Caused by: java.lang.ClassNotFoundException: com.couchbase.client.java.document.json.JsonObject 
     at java.net.URLClassLoader$1.run(URLClassLoader.java:366) 
     at java.net.URLClassLoader$1.run(URLClassLoader.java:355) 
     at java.security.AccessController.doPrivileged(Native Method) 
     at java.net.URLClassLoader.findClass(URLClassLoader.java:354) 
     at java.lang.ClassLoader.loadClass(ClassLoader.java:425) 
     at java.lang.ClassLoader.loadClass(ClassLoader.java:358) 
     ... 10 more 

내 받는다는 종속성은 다음과 같습니다 다음과 같은 예외가 점점 오전 : -

<dependency> 
     <groupId>org.apache.spark</groupId> 
     <artifactId>spark-core_2.10</artifactId> 
     <version>1.6.1</version> 
    </dependency> 
    <dependency> 
     <groupId>org.apache.spark</groupId> 
     <artifactId>spark-sql_2.10</artifactId> 
     <version>1.6.1</version> 
    </dependency> 
    <dependency> 
    <groupId>com.databricks</groupId> 
    <artifactId>spark-csv_2.10</artifactId> 
    <version>1.4.0</version> 
</dependency> 
<dependency> 
    <groupId>com.couchbase.client</groupId> 
    <artifactId>spark-connector_2.10</artifactId> 
    <version>1.1.0</version> 
</dependency> 
    <dependency> 
     <groupId>com.couchbase.client</groupId> 
     <artifactId>java-client</artifactId> 
     <version>2.3.4</version> 
    </dependency> 

내가 놓치고 곳을 가르쳐주세요.

Answer 1

다음은 저장하고 카우치베이스 주식회사에 JsonDocument를 검색하는 샘플 프로그램을 여기

<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-core_2.10</artifactId> 
    <version>1.6.2</version> 
</dependency> 
<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-sql_2.10</artifactId> 
    <version>1.6.2</version> 
</dependency> 
<dependency> 
    <groupId>com.couchbase.client</groupId> 
    <artifactId>spark-connector_2.10</artifactId> 
    <version>1.2.1</version> 
</dependency> 

그리고 스파크 1.6이다 사용 카우치베이스 주식회사에 연결하는 데 필요한 최소한의 종속성이 있습니다. 희망이 도움이됩니다.

import java.util.Arrays; 
import java.util.List; 
import org.apache.spark.SparkConf; 
import org.apache.spark.api.java.JavaRDD; 
import org.apache.spark.api.java.JavaSparkContext; 
import com.couchbase.client.java.document.JsonDocument; 
import com.couchbase.client.java.document.json.JsonObject; 
import com.couchbase.spark.japi.CouchbaseDocumentRDD; 
import com.couchbase.spark.japi.CouchbaseSparkContext; 

public class CouchBaseDemo { 
    public static void main(String[] args) { 
     //JavaSparkContext 
     SparkConf conf = new SparkConf().setAppName("CouchBaseDemo").setMaster("local").set("com.couchbase.bucket.travel-sample", ""); 
     JavaSparkContext jsc = new JavaSparkContext(conf); 
     CouchbaseSparkContext csc = CouchbaseSparkContext.couchbaseContext(jsc); 
     //Create and save JsonDocument 
     JsonDocument docOne = JsonDocument.create("docOne", JsonObject.create().put("new", "doc-content")); 
     JavaRDD<JsonDocument> jRDD = jsc.parallelize(Arrays.asList(docOne)); 
     CouchbaseDocumentRDD<JsonDocument> cbRDD = CouchbaseDocumentRDD.couchbaseDocumentRDD(jRDD); 
     cbRDD.saveToCouchbase(); 
     //fetch JsonDocument 
     List<JsonDocument> doc = csc.couchbaseGet(Arrays.asList("docOne")).collect(); 
     System.out.println(doc); 
    } 
}