0
IoT 센서 데이터의 이상 탐지를위한 자동 엔코더를 구현하고 있습니다. 내 데이터 세트는 시뮬레이션에서 나온 것이지만 기본적으로 가속도계 데이터입니다. 3 차원, 각 축마다 하나씩입니다. 내가 CSV 파일에서 읽고 있어요DeepLearning4J : FeedForward 자동 엔코더에서 모양이 일치하지 않습니다.
, 열 2-4 데이터를 포함 - 코드 품질에 대한 죄송합니다, 그것은 신속하고 더러운입니다 :
public static void main(String[] args) {
// Generate the training data
DataSetIterator iterator = getTrainingData(batchSize, rng);
// Create the network
int numInput = 3;
int numOutputs = 3;
int nHidden = 1;
int listenerFreq = batchSize/5;
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(seed)
.gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue)
.gradientNormalizationThreshold(1.0).iterations(iterations).momentum(0.5)
.momentumAfter(Collections.singletonMap(3, 0.9))
.optimizationAlgo(OptimizationAlgorithm.CONJUGATE_GRADIENT).list(2)
.layer(0,
new AutoEncoder.Builder().nIn(numInput).nOut(nHidden).weightInit(WeightInit.XAVIER)
.lossFunction(LossFunction.RMSE_XENT).corruptionLevel(0.3).build())
.layer(1, new OutputLayer.Builder(LossFunction.NEGATIVELOGLIKELIHOOD).activation("softmax").nIn(nHidden)
.nOut(numOutputs).build())
.pretrain(true).backprop(false).build();
MultiLayerNetwork model = new MultiLayerNetwork(conf);
model.init();
model.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(listenerFreq)));
for (int i = 0; i < nEpochs; i++) {
iterator.reset();
model.fit(iterator);
}
}
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
Reader in;
try {
in = new FileReader("/Users/romeokienzler/Downloads/lorenz_healthy.csv");
Iterable<CSVRecord> records;
records = CSVFormat.DEFAULT.parse(in);
int index = 0;
for (CSVRecord record : records) {
String[] recordArray = record.get(0).split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
이 순입니다
나는 다음과 같은 오류를 받고 있어요 : 모양이 일치하지 않습니다 : x.shape = [1, 9000], y.shape = [1, 3]
Exception in thread "main" java.lang.IllegalArgumentException: Shapes do not match: x.shape=[1, 9000], y.shape=[1, 3]
at org.nd4j.linalg.api.parallel.tasks.cpu.CPUTaskFactory.getTransformAction(CPUTaskFactory.java:92)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.doTransformOp(DefaultOpExecutioner.java:409)
at org.nd4j.linalg.api.ops.executioner.DefaultOpExecutioner.exec(DefaultOpExecutioner.java:62)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2660)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2641)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2419)
at org.deeplearning4j.nn.layers.feedforward.autoencoder.AutoEncoder.computeGradientAndScore(AutoEncoder.java:123)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:132)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.optimize(BaseOptimizer.java:151)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:52)
at org.deeplearning4j.nn.layers.BaseLayer.fit(BaseLayer.java:486)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.pretrain(MultiLayerNetwork.java:170)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1134)
at org.deeplearning4j
. 예. 피드 포워드. 자동 인코딩. 일반 검색기.). 내 실수는 어디 갔지?
미리 감사 ...
편집 : UPDATE 최신 버전 13.9.16 내가 같은 오류가 (의미) 받고 있어요에, 여기에 내가 지금 뭘하는지입니다 :
private static DataSetIterator getTrainingData(int batchSize, Random rand) {
double[] ix = new double[nSamples];
double[] iy = new double[nSamples];
double[] iz = new double[nSamples];
double[] ox = new double[nSamples];
double[] oy = new double[nSamples];
double[] oz = new double[nSamples];
try {
RandomAccessFile in = new RandomAccessFile(new File("/Users/romeokienzler/Downloads/lorenz_healthy.csv"),
"r");
int index = 0;
String record;
while ((record = in.readLine()) != null) {
String[] recordArray = record.split(";");
ix[index] = Double.parseDouble(recordArray[1]);
iy[index] = Double.parseDouble(recordArray[2]);
iz[index] = Double.parseDouble(recordArray[3]);
ox[index] = Double.parseDouble(recordArray[1]);
oy[index] = Double.parseDouble(recordArray[2]);
oz[index] = Double.parseDouble(recordArray[3]);
index++;
}
INDArray ixNd = Nd4j.create(ix);
INDArray iyNd = Nd4j.create(iy);
INDArray izNd = Nd4j.create(iz);
INDArray oxNd = Nd4j.create(ox);
INDArray oyNd = Nd4j.create(oy);
INDArray ozNd = Nd4j.create(oz);
INDArray iNd = Nd4j.hstack(ixNd, iyNd, izNd);
INDArray oNd = Nd4j.hstack(oxNd, oyNd, ozNd);
DataSet dataSet = new DataSet(iNd, oNd);
List<DataSet> listDs = dataSet.asList();
Collections.shuffle(listDs, rng);
return new ListDataSetIterator(listDs, batchSize);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(-1);
return null;
}
}
그리고 여기 순 :
// Set up network. 784 in/out (as MNIST images are 28x28).
// 784 -> 250 -> 10 -> 250 -> 784
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().seed(12345).iterations(1)
.weightInit(WeightInit.XAVIER).updater(Updater.ADAGRAD).activation("relu")
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).learningRate(learningRate)
.regularization(true).l2(0.0001).list().layer(0, new DenseLayer.Builder().nIn(3).nOut(1).build())
.layer(1, new OutputLayer.Builder().nIn(1).nOut(3).lossFunction(LossFunctions.LossFunction.MSE).build())
.pretrain(false).backprop(true).build();
MultiLayerNetwork net = new MultiLayerNetwork(conf);
net.setListeners(Collections.singletonList((IterationListener) new ScoreIterationListener(1)));
// Load data and split into training and testing sets. 40000 train,
// 10000 test
DataSetIterator iter = getTrainingData(batchSize, rng);
// Train model:
int nEpochs = 30;
while (iter.hasNext()) {
DataSet ds = iter.next();
for (int epoch = 0; epoch < nEpochs; epoch++) {
net.fit(ds.getFeatures(), ds.getLabels());
System.out.println("Epoch " + epoch + " complete");
}
}
내 오류는 다음과 같습니다
Exception in thread "main" java.lang.IllegalStateException: Mis matched lengths: [9000] != [3]
at org.nd4j.linalg.util.LinAlgExceptions.assertSameLength(LinAlgExceptions.java:39)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2786)
at org.nd4j.linalg.api.ndarray.BaseNDArray.subi(BaseNDArray.java:2767)
at org.nd4j.linalg.api.ndarray.BaseNDArray.sub(BaseNDArray.java:2547)
at org.deeplearning4j.nn.layers.BaseOutputLayer.getGradientsAndDelta(BaseOutputLayer.java:182)
at org.deeplearning4j.nn.layers.BaseOutputLayer.backpropGradient(BaseOutputLayer.java:161)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.calcBackpropGradients(MultiLayerNetwork.java:1125)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.backprop(MultiLayerNetwork.java:1077)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.computeGradientAndScore(MultiLayerNetwork.java:1817)
at org.deeplearning4j.optimize.solvers.BaseOptimizer.gradientAndScore(BaseOptimizer.java:152)
at org.deeplearning4j.optimize.solvers.StochasticGradientDescent.optimize(StochasticGradientDescent.java:54)
at org.deeplearning4j.optimize.Solver.optimize(Solver.java:51)
at org.deeplearning4j.nn.multilayer.MultiLayerNetwork.fit(MultiLayerNetwork.java:1445)
at org.deeplearning4j.examples.feedforward.anomalydetection.IoTAnomalyExample.main(IoTAnomalyExample.java:110)
나는 훈련 데이터를 엉망으로 만들고 있다고 확신한다. 훈련 데이터의 모양은 목표 (동일한 데이터는 자동 코드 작성기를 만들고 싶다)에 대해 3000 행 3 열이다. 테스트 데이터는 여기에서 찾을 수 있습니다 : https://pmqsimulator-romeokienzler-2310.mybluemix.net/data
어떤 아이디어? Skymind의 Alex Black에
첫째, 왜 rc3.8를 실행하지 않도록 (모양의 잘못을 가지고). 우리는 1 월에 자바 물건을 제거했습니다 ... 우리는 지금 힘으로 달립니다 (그리고 몇 달 동안 가지고 있습니다) 먼저 업그레이드를 시도하십시오. 이제는 그 버전의 지원을 정당화하는 것이 어렵습니다. –