java解析超大json文件数据

        json文件大小为10G左右，对于这种超大的json使用jackson的objectmapper一把映射到java对象是不太可行的，内存一般都会溢出。此时需要使用jackson的JsonParser从文件中逐个token去读取。一般大json中都会存在某个数组中有超多的数据记录，我们需要解决的就是记录当前token路径，在读取到超大json数组时，再利用逐条数据读取mapper.readTree(jsonParser)逐条读取数据，利用数组缓存一定量的数据后，写入数据库后继续读取，知道json数组数据读取结束。

        这里关键的是记录当前json的token的路径，以方便地利用mapper.readTree(jsonParser)读取任意的子节点数据。

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import sarifreport.Result;
 
import java.io.File;
import java.io.IOException;
 
public class Main {
    public static void main(String [] args){
        long start = System.currentTimeMillis();
        try {
            ObjectMapper mapper = new ObjectMapper();
            JsonFactory jsonFactory = new JsonFactory();
            JsonParser jsonParser = jsonFactory.createParser(
                    new File("E:/result2.json"));
            JsonPath path = new JsonPath();
            while (jsonParser.nextToken() != null) {
                traverse(jsonParser, 0,path,mapper);
            }
            jsonParser.close();
            System.out.println((System.currentTimeMillis()-start)/1000+"秒");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    private static void traverse(JsonParser jsonParser, int depth, JsonPath path, ObjectMapper mapper) throws IOException {
        JsonToken token = jsonParser.getCurrentToken();
 
        if (token == null) {
            return;
        }
        setPath(jsonParser,depth,path);
        if (path.toString().equals("root.runs[i].results[i]")) {
            while (jsonParser.nextToken() == JsonToken.START_OBJECT) {
                JsonNode jsonNode = mapper.readTree(jsonParser);
                // 此处可以读取一定数量的数据后入库
                System.out.println(jsonNode.toPrettyString());
            }
            token = jsonParser.getCurrentToken();
        }
        // Recursive call for nested structures
        if (token == JsonToken.START_OBJECT || token == JsonToken.START_ARRAY) {
            while (jsonParser.nextToken() != null && jsonParser.getCurrentToken() != JsonToken.END_OBJECT
                    && jsonParser.getCurrentToken() != JsonToken.END_ARRAY) {
                traverse(jsonParser, depth + 1,path,mapper);
            }
        }
    }
 
    /**
     * 同一个depth只记录一个node
     * @param jsonParser
     * @param depth
     * @param path
     * @throws IOException
     */
    private static void  setPath(JsonParser jsonParser, int depth, JsonPath path) throws IOException {
        JsonToken token = jsonParser.getCurrentToken();
        if (path.nodeLink.size() > depth+1) {
            path.nodeLink.subList(depth+1,path.nodeLink.size()).clear();
        }
        JsonPath.Node prefixNode = path.nodeLink.size()>=(depth+1)?path.nodeLink.get(depth):null;
        switch (token) {
            case FIELD_NAME:
                if (prefixNode != null) {
                    prefixNode.nodeType = JsonPath.NODE_FIELD;
                    prefixNode.nodeName = jsonParser.getCurrentName();
                } else {
                    JsonPath.Node node = new JsonPath.Node();
                    node.nodeType = JsonPath.NODE_FIELD;
                    node.nodeName = jsonParser.getCurrentName();
                    path.nodeLink.add(node);
                }
                break;
            case START_ARRAY:
                if (prefixNode != null && JsonPath.NODE_FIELD.equals(prefixNode.nodeType)) {
                    prefixNode.nodeName = prefixNode.nodeName+"[i]";
                } else {
                    JsonPath.Node node = new JsonPath.Node();
                    node.nodeType = JsonPath.NODE_ARRAY;
                    path.nodeLink.add(node);
                }
                break;
            case START_OBJECT:
                if (prefixNode != null && JsonPath.NODE_FIELD.equals(prefixNode.nodeType)) {
                    prefixNode.nodeName = prefixNode.nodeName+".";
                } else {
                    JsonPath.Node node = new JsonPath.Node();
                    node.nodeType = JsonPath.NODE_OBJECT;
                    path.nodeLink.add(node);
                }
                break;
        }
    }
}复制

import java.util.LinkedList;
 
public class JsonPath {
    public static String NODE_ARRAY = "array";
    public static String NODE_OBJECT = "object";
    public static String NODE_FIELD = "field";
    public LinkedList<Node> nodeLink = new LinkedList<>();
 
    public static class Node{
        public String nodeName;
        public String nodeType;
    }
 
    @Override
    public String toString() {
        if (this.nodeLink == null || nodeLink.isEmpty()) {
            return "";
        }
        StringBuilder path = new StringBuilder();
        for (int i = 0; i < nodeLink.size(); i++ ) {
            Node node = this.nodeLink.get(i);
            if (node == null) {
                continue;
            }
            if (i == 0) {
                if (NODE_ARRAY.equals(node.nodeType)) {
                    path.append("root[i]");
                } else {
                    path.append("root");
                }
            }
            if (NODE_ARRAY.equals(node.nodeType)) {
                path.append("[i]");
            }else if (NODE_OBJECT.equals(node.nodeType)) {
                path.append(".");
            } else if (NODE_FIELD.equals(node.nodeType)) {
                path.append(node.nodeName);
            }
        }
        return path.toString();
    }
}复制

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
 
    <groupId>org.example</groupId>
    <artifactId>BigJsonDeal</artifactId>
    <version>1.0-SNAPSHOT</version>
 
    <properties>
        <maven.compiler.source>19</maven.compiler.source>
        <maven.compiler.target>19</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>
 
    <dependencies>
        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.16.1</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.jsonschema2pojo</groupId>
                <artifactId>jsonschema2pojo-maven-plugin</artifactId>
                <version>1.2.1</version>
                <configuration>
                    <sourceDirectory>${basedir}/src/main/resources/schema</sourceDirectory>
                    <targetProject>${basedir}/src/main/java</targetProject>
                    <targetPackage>com.example.types</targetPackage>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>复制

 jsonschema2pojo为一个根据json结构生成java的pojo代码的工具，对于复杂json生成pojo十分省时间。