SpringBoot整合Java DL4J实现文本分类系统

<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-core</artifactId>
    <version>1.0.0-beta7</version>
</dependency>
<dependency>
    <groupId>org.deeplearning4j</groupId>
    <artifactId>deeplearning4j-nlp</artifactId>
    <version>1.0.0-beta7</version>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
</dependency>

import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory;
import org.deeplearning4j.text.tokenization.tokenizerfactory.UimaTokenizerFactory;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.preprocessor.DataNormalization;
import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize;

public class DataPreprocessor {

    public static DataSetIterator preprocessData(String filePath) {
        // 创建 TokenizerFactory
        TokenizerFactory tokenizerFactory = new UimaTokenizerFactory();

        // 创建文档向量器
        DocumentVectorizer documentVectorizer = new DocumentVectorizer.Builder()
               .setTokenizerFactory(tokenizerFactory)
               .build();

        // 加载数据集
        InMemoryDataSetIterator dataSetIterator = new InMemoryDataSetIterator.Builder()
               .addSource(filePath, documentVectorizer)
               .build();

        // 数据标准化
        DataNormalization normalizer = new NormalizerStandardize();
        normalizer.fit(dataSetIterator);
        dataSetIterator.setPreProcessor(normalizer);

        return dataSetIterator;
    }
}

import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.layers.LSTM;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.lossfunctions.LossFunctions;

public class TextClassificationModel {

    public static MultiLayerNetwork buildModel(int inputSize, int numClasses) {
        // 构建神经网络配置
        MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder()
               .seed(12345)
               .weightInit(WeightInit.XAVIER)
               .updater(Updater.ADAGRAD)
               .list()
               .layer(0, new LSTM.Builder()
                       .nIn(inputSize)
                       .nOut(128)
                       .activation(Activation.TANH)
                       .build())
               .layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.NEGATIVELOGLIKELIHOOD)
                       .activation(Activation.SOFTMAX)
                       .nOut(numClasses)
                       .build())
               .build();

        // 创建神经网络模型
        MultiLayerNetwork model = new MultiLayerNetwork(configuration);
        model.init();

        return model;
    }
}

import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;

public class ModelTrainer {

    public static void trainModel(MultiLayerNetwork model, DataSetIterator iterator, int numEpochs) {
        // 设置优化算法和学习率
        model.setOptimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT);
        model.setLearningRate(0.01);

        // 添加训练监听器
        model.setListeners(new ScoreIterationListener(100));

        // 训练模型
        for (int epoch = 0; epoch < numEpochs; epoch++) {
            model.fit(iterator);
            System.out.println("Epoch " + epoch + " completed.");
        }
    }
}

import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;

public class ModelPredictor {

    public static String predictCategory(MultiLayerNetwork model, String text) {
        // 预处理文本数据
        DataSet dataSet = DataPreprocessor.preprocessData(text);

        // 预测类别
        INDArray output = model.output(dataSet.getFeatureMatrix());
        int predictedClass = argMax(output);

        // 返回类别名称
        return getCategoryName(predictedClass);
    }

    private static int argMax(INDArray array) {
        double maxValue = Double.NEGATIVE_INFINITY;
        int maxIndex = -1;
        for (int i = 0; i < array.length(); i++) {
            if (array.getDouble(i) > maxValue) {
                maxValue = array.getDouble(i);
                maxIndex = i;
            }
        }
        return maxIndex;
    }

    private static String getCategoryName(int classIndex) {
        // 根据类别索引返回类别名称
        switch (classIndex) {
            case 0:
                return "政治新闻";
            case 1:
                return "体育新闻";
            case 2:
                return "娱乐新闻";
            default:
                return "未知类别";
        }
    }
}

import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;

import static org.junit.jupiter.api.Assertions.assertEquals;

public class TextClassificationSystemTest {

    private MultiLayerNetwork model;
    private DataSetIterator iterator;

    @BeforeEach
    public void setUp() {
        // 加载数据集并预处理
        iterator = DataPreprocessor.preprocessData("path/to/dataset.csv");

        // 构建模型
        model = TextClassificationModel.buildModel(iterator.inputColumns(), iterator.totalOutcomes());
    }

    @Test
    public void testModelTraining() {
        // 训练模型
        ModelTrainer.trainModel(model, iterator, 10);

        // 预测结果
        String text = "美国总统拜登发表重要讲话";
        String predictedCategory = ModelPredictor.predictCategory(model, text);

        // 验证预测结果
        assertEquals("政治新闻", predictedCategory);
    }
}

Epoch 0 completed.
Epoch 1 completed.
...
Epoch 9 completed.